



import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece







class       _snake_case				(     unittest.TestCase ):






   '''simple docstring'''






   def       A__    (    self:     Dict  ,lowerCamelCase_:     Tuple							)	->      Any:
       UpperCAmelCase_     :       Tuple  		=					list(tokenizer_class.max_model_input_sizes.keys()							)
       for model_name in sa_models[:1]:
           UpperCAmelCase_     :       Any  		=					tokenizer_class.from_pretrained(snake_case__							)
           self.assertIsNotNone(snake_case__							)
           self.assertIsInstance(snake_case__  ,snake_case__							)
           self.assertIsInstance(snake_case__  ,snake_case__							)

           for special_tok in tokenizer.all_special_tokens:
               self.assertIsInstance(snake_case__  ,snake_case__							)
               UpperCAmelCase_     :       List[str]  		=					tokenizer.convert_tokens_to_ids(snake_case__							)
               self.assertIsInstance(snake_case__  ,snake_case__							)


   def       A__    (    self:     Tuple  ,lowerCamelCase_:     BatchEncoding  ,lowerCamelCase_:     Optional[Callable] = None							)	->      List[str]:
       UpperCAmelCase_     :       Union[str, Any]  		=					pickle.dumps(snake_case__							)
       self.assertIsNotNone(snake_case__							)

       UpperCAmelCase_     :       Dict  		=					pickle.loads(snake_case__							)

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast  ,be_original.is_fast							)

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
           self.assertIsNotNone(be_restored.encodings							)
       else:
           self.assertIsNone(be_restored.encodings							)

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()  ,be_restored.values()							):
           if equal_op:
               self.assertTrue(equal_op(snake_case__  ,snake_case__							)							)
           else:
               self.assertEqual(snake_case__  ,snake_case__							)


   @slow
   def       A__    (    self:     Optional[Any]							)	->      Dict:
       self.check_tokenizer_from_pretrained(snake_case__							)


   def       A__    (    self:     int							)	->      Optional[int]:
       self.assertEqual(TensorType("""tf"""							)  ,TensorType.TENSORFLOW							)
       self.assertEqual(TensorType("""pt"""							)  ,TensorType.PYTORCH							)
       self.assertEqual(TensorType("""np"""							)  ,TensorType.NUMPY							)


   @require_tokenizers
   def       A__    (    self:     Optional[Any]							)	->      Union[str, Any]:
       import numpy as np

       UpperCAmelCase_     :       Union[str, Any]  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)
       UpperCAmelCase_     :       List[Any]  		=					BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

       # Python no tensor
       with self.subTest("""BatchEncoding (Python, return_tensors=None)"""							):
           self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""							)							)

       with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""							):
           self.assert_dump_and_restore(
               tokenizer_p("""Small example to encode"""  ,return_tensors=TensorType.NUMPY							)  ,np.array_equal							)

       with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""							):
           self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""							)							)

       with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""							):
           self.assert_dump_and_restore(
               tokenizer_r("""Small example to encode"""  ,return_tensors=TensorType.NUMPY							)  ,np.array_equal							)


   @require_tf
   @require_tokenizers
   def       A__    (    self:     Dict							)	->      str:
       import tensorflow as tf

       def tf_array_equals(lowerCamelCase_:     List[Any]  ,lowerCamelCase_:     str							):
           return tf.reduce_all(tf.equal(snake_case__  ,snake_case__							)							)

       UpperCAmelCase_     :       Tuple  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)
       UpperCAmelCase_     :       Dict  		=					BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

       with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""							):
           self.assert_dump_and_restore(
               tokenizer_p("""Small example to encode"""  ,return_tensors=TensorType.TENSORFLOW							)  ,snake_case__							)

       with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""							):
           self.assert_dump_and_restore(
               tokenizer_r("""Small example to encode"""  ,return_tensors=TensorType.TENSORFLOW							)  ,snake_case__							)


   @require_torch
   @require_tokenizers
   def       A__    (    self:     Optional[int]							)	->      List[str]:
       import torch

       UpperCAmelCase_     :       int  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)
       UpperCAmelCase_     :       Optional[Any]  		=					BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

       with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""							):
           self.assert_dump_and_restore(
               tokenizer_p("""Small example to encode"""  ,return_tensors=TensorType.PYTORCH							)  ,torch.equal							)

       with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""							):
           self.assert_dump_and_restore(
               tokenizer_r("""Small example to encode"""  ,return_tensors=TensorType.PYTORCH							)  ,torch.equal							)


   @require_tokenizers
   def       A__    (    self:     Optional[Any]							)	->      List[str]:
       UpperCAmelCase_     :       List[str]  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)
       UpperCAmelCase_     :       Optional[int]  		=					BertTokenizerFast.from_pretrained("""bert-base-cased"""							)

       with self.subTest("""Python Tokenizer"""							):
           self.assertFalse(tokenizer_p("""Small example to_encode"""							).is_fast							)

       with self.subTest("""Rust Tokenizer"""							):
           self.assertTrue(tokenizer_r("""Small example to_encode"""							).is_fast							)


   @require_tokenizers
   def       A__    (    self:     Optional[int]							)	->      Optional[int]:
       UpperCAmelCase_     :       int  		=					BertTokenizerFast.from_pretrained("""bert-base-cased"""							)
       UpperCAmelCase_     :       Union[str, Any]  		=					tokenizer_r(["""Test""", """\xad""", """test"""]  ,is_split_into_words=snake_case__							)

       self.assertEqual(encoded.word_to_tokens(0							)  ,TokenSpan(start=1  ,end=2							)							)
       self.assertEqual(encoded.word_to_tokens(1							)  ,snake_case__							)
       self.assertEqual(encoded.word_to_tokens(2							)  ,TokenSpan(start=2  ,end=3							)							)


   def       A__    (    self:     Optional[Any]							)	->      Any:
       UpperCAmelCase_     :       List[Any]  		=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
       UpperCAmelCase_     :       List[Any]  		=					batch.convert_to_tensors(tensor_type="""np"""							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(2, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(2,)							)
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCAmelCase_     :       Any  		=					batch.convert_to_tensors(tensor_type="""np"""							)
       self.assertFalse(len(cs.err							)  ,msg=F'''should have no warning, but got {cs.err}'''							)

       UpperCAmelCase_     :       Tuple  		=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
       UpperCAmelCase_     :       str  		=					batch.convert_to_tensors(tensor_type="""np"""  ,prepend_batch_axis=snake_case__							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(1, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(1,)							)


   @require_torch
   def       A__    (    self:     Tuple							)	->      int:
       UpperCAmelCase_     :       List[str]  		=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
       UpperCAmelCase_     :       Optional[int]  		=					batch.convert_to_tensors(tensor_type="""pt"""							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(2, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(2,)							)
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCAmelCase_     :       List[Any]  		=					batch.convert_to_tensors(tensor_type="""pt"""							)
       self.assertFalse(len(cs.err							)  ,msg=F'''should have no warning, but got {cs.err}'''							)

       UpperCAmelCase_     :       int  		=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
       UpperCAmelCase_     :       Tuple  		=					batch.convert_to_tensors(tensor_type="""pt"""  ,prepend_batch_axis=snake_case__							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(1, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(1,)							)


   @require_tf
   def       A__    (    self:     Tuple							)	->      Tuple:
       UpperCAmelCase_     :       Dict  		=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
       UpperCAmelCase_     :       Tuple  		=					batch.convert_to_tensors(tensor_type="""tf"""							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(2, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(2,)							)
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCAmelCase_     :       List[str]  		=					batch.convert_to_tensors(tensor_type="""tf"""							)
       self.assertFalse(len(cs.err							)  ,msg=F'''should have no warning, but got {cs.err}'''							)

       UpperCAmelCase_     :       Optional[int]  		=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
       UpperCAmelCase_     :       Dict  		=					batch.convert_to_tensors(tensor_type="""tf"""  ,prepend_batch_axis=snake_case__							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(1, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(1,)							)


   @require_flax
   def       A__    (    self:     List[Any]							)	->      Optional[Any]:
       UpperCAmelCase_     :       Any  		=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}							)
       UpperCAmelCase_     :       List[Any]  		=					batch.convert_to_tensors(tensor_type="""jax"""							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(2, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(2,)							)
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCAmelCase_     :       Union[str, Any]  		=					batch.convert_to_tensors(tensor_type="""jax"""							)
       self.assertFalse(len(cs.err							)  ,msg=F'''should have no warning, but got {cs.err}'''							)

       UpperCAmelCase_     :       str  		=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}							)
       UpperCAmelCase_     :       Optional[int]  		=					batch.convert_to_tensors(tensor_type="""jax"""  ,prepend_batch_axis=snake_case__							)
       self.assertEqual(tensor_batch["""inputs"""].shape  ,(1, 3)							)
       self.assertEqual(tensor_batch["""labels"""].shape  ,(1,)							)


   def       A__    (    self:     List[str]							)	->      Dict:
       UpperCAmelCase_     :       Tuple  		=					[{"""input_ids""": np.array([0, 1, 2]							)}, {"""input_ids""": np.array([0, 1, 2, 3]							)}]
       UpperCAmelCase_     :       List[Any]  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)

       UpperCAmelCase_     :       List[str]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,np.ndarray							)							)
       self.assertEqual(batch["""input_ids"""].tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
       UpperCAmelCase_     :       Optional[Any]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__  ,return_tensors="""np"""							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,np.ndarray							)							)
       self.assertEqual(batch["""input_ids"""].tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)


   @require_torch
   def       A__    (    self:     List[Any]							)	->      int:
       import torch

       UpperCAmelCase_     :       Optional[Any]  		=					[{"""input_ids""": torch.tensor([0, 1, 2]							)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]							)}]
       UpperCAmelCase_     :       List[str]  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)

       UpperCAmelCase_     :       List[str]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,torch.Tensor							)							)
       self.assertEqual(batch["""input_ids"""].tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
       UpperCAmelCase_     :       List[str]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__  ,return_tensors="""pt"""							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,torch.Tensor							)							)
       self.assertEqual(batch["""input_ids"""].tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)


   @require_tf
   def       A__    (    self:     int							)	->      Union[str, Any]:
       import tensorflow as tf

       UpperCAmelCase_     :       Dict  		=					[{"""input_ids""": tf.constant([0, 1, 2]							)}, {"""input_ids""": tf.constant([0, 1, 2, 3]							)}]
       UpperCAmelCase_     :       Optional[int]  		=					BertTokenizer.from_pretrained("""bert-base-cased"""							)

       UpperCAmelCase_     :       Optional[Any]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,tf.Tensor							)							)
       self.assertEqual(batch["""input_ids"""].numpy().tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
       UpperCAmelCase_     :       Optional[int]  		=					tokenizer.pad(snake_case__  ,padding=snake_case__  ,return_tensors="""tf"""							)
       self.assertTrue(isinstance(batch["""input_ids"""]  ,tf.Tensor							)							)
       self.assertEqual(batch["""input_ids"""].numpy().tolist()  ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)


   @require_tokenizers
   def       A__    (    self:     Any							)	->      List[Any]:
       UpperCAmelCase_     :       int  		=					Tokenizer(WordPiece(unk_token="""[UNK]"""							)							)
       PreTrainedTokenizerFast(tokenizer_object=snake_case__							)






   @require_tokenizers
   def       A__    (    self:     int							)	->      Any:
       UpperCAmelCase_     :       str  		=					Tokenizer(WordPiece(unk_token="""[UNK]"""							)							)
       with tempfile.TemporaryDirectory() as tmpdirname:
           bert_tokenizer.save(os.path.join(snake_case__  ,"""tokenizer.json"""							)							)
           PreTrainedTokenizerFast(tokenizer_file=os.path.join(snake_case__  ,"""tokenizer.json"""							)							)






def 				__SCREAMING_SNAKE_CASE    (     snake_case_		):

						'''simple docstring'''


						return int(snake_case_		) if x - int(snake_case_		) >= 0 else int(snake_case_		) - 1


if __name__ == "__main__":
				import doctest

				doctest.testmod()




