


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
       from tokenizers import Tokenizer
       from tokenizers.models import WordPiece

class    __magic_name__						(  unittest.TestCase):


     def   UpperCAmelCase__						(   self				:  int				,						lowerCamelCase__				:  Any      )	->   List[str]:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[Any] 							= list(tokenizer_class.max_model_input_sizes.keys()      )
      for model_name in sa_models[:1]:
       UpperCamelCase__       :     Union[str, Any] 							= tokenizer_class.from_pretrained(_lowercase      )
       self.assertIsNotNone(_lowercase      )
       self.assertIsInstance(_lowercase				,						_lowercase      )
       self.assertIsInstance(_lowercase				,						_lowercase      )

       for special_tok in tokenizer.all_special_tokens:
        self.assertIsInstance(_lowercase				,						_lowercase      )
        UpperCamelCase__       :     str 							= tokenizer.convert_tokens_to_ids(_lowercase      )
        self.assertIsInstance(_lowercase				,						_lowercase      )


     def   UpperCAmelCase__						(   self				:  Any				,						lowerCamelCase__				:  int				,						lowerCamelCase__				:  Union[str, Any] = None      )	->   Any:





      '''simple docstring'''






      UpperCamelCase__       :     str 							= pickle.dumps(_lowercase      )
      self.assertIsNotNone(_lowercase      )

      UpperCamelCase__       :     Dict 							= pickle.loads(_lowercase      )

      # Ensure is_fast is correctly restored
      self.assertEqual(be_restored.is_fast				,						be_original.is_fast      )

      # Ensure encodings are potentially correctly restored
      if be_original.is_fast:
       self.assertIsNotNone(be_restored.encodings      )
      else:
       self.assertIsNone(be_restored.encodings      )

      # Ensure the keys are the same
      for original_v, restored_v in zip(be_original.values()				,						be_restored.values()      ):
       if equal_op:
        self.assertTrue(equal_op(_lowercase				,						_lowercase      )      )
       else:
        self.assertEqual(_lowercase				,						_lowercase      )


     @slow
     def   UpperCAmelCase__						(   self				:  int      )	->   Tuple:





      '''simple docstring'''






      self.check_tokenizer_from_pretrained(_lowercase      )


     def   UpperCAmelCase__						(   self				:  Optional[int]      )	->   str:





      '''simple docstring'''






      self.assertEqual(TensorType('''tf'''      )				,						TensorType.TENSORFLOW      )
      self.assertEqual(TensorType('''pt'''      )				,						TensorType.PYTORCH      )
      self.assertEqual(TensorType('''np'''      )				,						TensorType.NUMPY      )


     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  Dict      )	->   Tuple:





      '''simple docstring'''






      import numpy as np

      UpperCamelCase__       :     Optional[int] 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )
      UpperCamelCase__       :     Tuple 							= BertTokenizerFast.from_pretrained('''bert-base-cased'''      )

      # Python no tensor
      with self.subTest('''BatchEncoding (Python, return_tensors=None)'''      ):
       self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''      )      )

      with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''      ):
       self.assert_dump_and_restore(
           tokenizer_p('''Small example to encode'''				,						return_tensors=TensorType.NUMPY      )				,						np.array_equal      )

      with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''      ):
       self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''      )      )

      with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''      ):
       self.assert_dump_and_restore(
           tokenizer_r('''Small example to encode'''				,						return_tensors=TensorType.NUMPY      )				,						np.array_equal      )


     @require_tf
     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  List[Any]      )	->   List[str]:





      '''simple docstring'''






      import tensorflow as tf

      def tf_array_equals(lowerCamelCase__				:  List[str]				,						lowerCamelCase__				:  Union[str, Any]      ):
       return tf.reduce_all(tf.equal(_lowercase				,						_lowercase      )      )

      UpperCamelCase__       :     List[str] 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )
      UpperCamelCase__       :     int 							= BertTokenizerFast.from_pretrained('''bert-base-cased'''      )

      with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''      ):
       self.assert_dump_and_restore(
           tokenizer_p('''Small example to encode'''				,						return_tensors=TensorType.TENSORFLOW      )				,						_lowercase      )

      with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''      ):
       self.assert_dump_and_restore(
           tokenizer_r('''Small example to encode'''				,						return_tensors=TensorType.TENSORFLOW      )				,						_lowercase      )


     @require_torch
     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  Union[str, Any]      )	->   List[Any]:





      '''simple docstring'''






      import torch

      UpperCamelCase__       :     Optional[Any] 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )
      UpperCamelCase__       :     Tuple 							= BertTokenizerFast.from_pretrained('''bert-base-cased'''      )

      with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''      ):
       self.assert_dump_and_restore(
           tokenizer_p('''Small example to encode'''				,						return_tensors=TensorType.PYTORCH      )				,						torch.equal      )

      with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''      ):
       self.assert_dump_and_restore(
           tokenizer_r('''Small example to encode'''				,						return_tensors=TensorType.PYTORCH      )				,						torch.equal      )


     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  Tuple      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     str 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )
      UpperCamelCase__       :     str 							= BertTokenizerFast.from_pretrained('''bert-base-cased'''      )

      with self.subTest('''Python Tokenizer'''      ):
       self.assertFalse(tokenizer_p('''Small example to_encode'''      ).is_fast      )

      with self.subTest('''Rust Tokenizer'''      ):
       self.assertTrue(tokenizer_r('''Small example to_encode'''      ).is_fast      )


     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  List[str]      )	->   int:





      '''simple docstring'''






      UpperCamelCase__       :     Any 							= BertTokenizerFast.from_pretrained('''bert-base-cased'''      )
      UpperCamelCase__       :     int 							= tokenizer_r(['''Test''', '''\xad''', '''test''']				,						is_split_into_words=_lowercase      )

      self.assertEqual(encoded.word_to_tokens(0      )				,						TokenSpan(start=1				,						end=2      )      )
      self.assertEqual(encoded.word_to_tokens(1      )				,						_lowercase      )
      self.assertEqual(encoded.word_to_tokens(2      )				,						TokenSpan(start=2				,						end=3      )      )


     def   UpperCAmelCase__						(   self				:  Tuple      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}      )
      UpperCamelCase__       :     Optional[Any] 							= batch.convert_to_tensors(tensor_type='''np'''      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(2, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(2,)      )
      # test converting the converted
      with CaptureStderr() as cs:
       UpperCamelCase__       :     Union[str, Any] 							= batch.convert_to_tensors(tensor_type='''np'''      )
      self.assertFalse(len(cs.err      )				,						msg=F"should have no warning, but got {cs.err}"      )

      UpperCamelCase__       :     int 							= BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}      )
      UpperCamelCase__       :     Dict 							= batch.convert_to_tensors(tensor_type='''np'''				,						prepend_batch_axis=_lowercase      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(1, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(1,)      )


     @require_torch
     def   UpperCAmelCase__						(   self				:  int      )	->   List[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     int 							= BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}      )
      UpperCamelCase__       :     Optional[int] 							= batch.convert_to_tensors(tensor_type='''pt'''      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(2, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(2,)      )
      # test converting the converted
      with CaptureStderr() as cs:
       UpperCamelCase__       :     List[Any] 							= batch.convert_to_tensors(tensor_type='''pt'''      )
      self.assertFalse(len(cs.err      )				,						msg=F"should have no warning, but got {cs.err}"      )

      UpperCamelCase__       :     List[Any] 							= BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}      )
      UpperCamelCase__       :     Dict 							= batch.convert_to_tensors(tensor_type='''pt'''				,						prepend_batch_axis=_lowercase      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(1, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(1,)      )


     @require_tf
     def   UpperCAmelCase__						(   self				:  str      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}      )
      UpperCamelCase__       :     List[Any] 							= batch.convert_to_tensors(tensor_type='''tf'''      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(2, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(2,)      )
      # test converting the converted
      with CaptureStderr() as cs:
       UpperCamelCase__       :     int 							= batch.convert_to_tensors(tensor_type='''tf'''      )
      self.assertFalse(len(cs.err      )				,						msg=F"should have no warning, but got {cs.err}"      )

      UpperCamelCase__       :     int 							= BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}      )
      UpperCamelCase__       :     Optional[int] 							= batch.convert_to_tensors(tensor_type='''tf'''				,						prepend_batch_axis=_lowercase      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(1, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(1,)      )


     @require_flax
     def   UpperCAmelCase__						(   self				:  List[str]      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     Union[str, Any] 							= BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}      )
      UpperCamelCase__       :     Tuple 							= batch.convert_to_tensors(tensor_type='''jax'''      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(2, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(2,)      )
      # test converting the converted
      with CaptureStderr() as cs:
       UpperCamelCase__       :     Union[str, Any] 							= batch.convert_to_tensors(tensor_type='''jax'''      )
      self.assertFalse(len(cs.err      )				,						msg=F"should have no warning, but got {cs.err}"      )

      UpperCamelCase__       :     Union[str, Any] 							= BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}      )
      UpperCamelCase__       :     Optional[int] 							= batch.convert_to_tensors(tensor_type='''jax'''				,						prepend_batch_axis=_lowercase      )
      self.assertEqual(tensor_batch['''inputs'''].shape				,						(1, 3)      )
      self.assertEqual(tensor_batch['''labels'''].shape				,						(1,)      )


     def   UpperCAmelCase__						(   self				:  Dict      )	->   Dict:





      '''simple docstring'''






      UpperCamelCase__       :     List[Any] 							= [{"""input_ids""": np.array([0, 1, 2]      )}, {"""input_ids""": np.array([0, 1, 2, 3]      )}]
      UpperCamelCase__       :     Optional[Any] 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )

      UpperCamelCase__       :     int 							= tokenizer.pad(_lowercase				,						padding=_lowercase      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						np.ndarray      )      )
      self.assertEqual(batch['''input_ids'''].tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
      UpperCamelCase__       :     Optional[int] 							= tokenizer.pad(_lowercase				,						padding=_lowercase				,						return_tensors='''np'''      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						np.ndarray      )      )
      self.assertEqual(batch['''input_ids'''].tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )


     @require_torch
     def   UpperCAmelCase__						(   self				:  Dict      )	->   Union[str, Any]:





      '''simple docstring'''






      import torch

      UpperCamelCase__       :     Optional[int] 							= [{"""input_ids""": torch.tensor([0, 1, 2]      )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]      )}]
      UpperCamelCase__       :     int 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )

      UpperCamelCase__       :     List[Any] 							= tokenizer.pad(_lowercase				,						padding=_lowercase      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						torch.Tensor      )      )
      self.assertEqual(batch['''input_ids'''].tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
      UpperCamelCase__       :     int 							= tokenizer.pad(_lowercase				,						padding=_lowercase				,						return_tensors='''pt'''      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						torch.Tensor      )      )
      self.assertEqual(batch['''input_ids'''].tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )


     @require_tf
     def   UpperCAmelCase__						(   self				:  Tuple      )	->   Dict:





      '''simple docstring'''






      import tensorflow as tf

      UpperCamelCase__       :     int 							= [{"""input_ids""": tf.constant([0, 1, 2]      )}, {"""input_ids""": tf.constant([0, 1, 2, 3]      )}]
      UpperCamelCase__       :     int 							= BertTokenizer.from_pretrained('''bert-base-cased'''      )

      UpperCamelCase__       :     List[str] 							= tokenizer.pad(_lowercase				,						padding=_lowercase      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						tf.Tensor      )      )
      self.assertEqual(batch['''input_ids'''].numpy().tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )
      UpperCamelCase__       :     List[str] 							= tokenizer.pad(_lowercase				,						padding=_lowercase				,						return_tensors='''tf'''      )
      self.assertTrue(isinstance(batch['''input_ids''']				,						tf.Tensor      )      )
      self.assertEqual(batch['''input_ids'''].numpy().tolist()				,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]      )


     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  Optional[int]      )	->   Tuple:





      '''simple docstring'''






      UpperCamelCase__       :     Optional[int] 							= Tokenizer(WordPiece(unk_token='''[UNK]'''      )      )
      PreTrainedTokenizerFast(tokenizer_object=_lowercase      )


     @require_tokenizers
     def   UpperCAmelCase__						(   self				:  Tuple      )	->   Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__       :     int 							= Tokenizer(WordPiece(unk_token='''[UNK]'''      )      )
      with tempfile.TemporaryDirectory() as tmpdirname:
       bert_tokenizer.save(os.path.join(_lowercase				,						'''tokenizer.json'''      )      )
       PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase				,						'''tokenizer.json'''      )      )









def 				_lowerCAmelCase			(	_a :       str      )						->      int:
     lowerCAmelCase_       :							List[str]			 =		53_81
     for x in s:
          lowerCAmelCase_       :							Dict			 =		((hash_value << 5) + hash_value) + ord(_a      )
     return hash_value & 0XFFFFFFFF
