import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece



class     snake_case_			(						unittest.TestCase			):






   def  __UpperCamelCase	( self							:	Any      ,		lowercase_							:	Dict    )   ->		List[str]:
       lowercase__					:  Optional[int]          =							list(tokenizer_class.max_model_input_sizes.keys()    )
       for model_name in sa_models[:1]:
           lowercase__					:  str          =							tokenizer_class.from_pretrained(lowercase_    )
           self.assertIsNotNone(lowercase_    )
           self.assertIsInstance(lowercase_      ,		lowercase_    )
           self.assertIsInstance(lowercase_      ,		lowercase_    )

           for special_tok in tokenizer.all_special_tokens:
               self.assertIsInstance(lowercase_      ,		lowercase_    )
               lowercase__					:  List[str]          =							tokenizer.convert_tokens_to_ids(lowercase_    )
               self.assertIsInstance(lowercase_      ,		lowercase_    )






   def  __UpperCamelCase	( self							:	Dict      ,		lowercase_							:	BatchEncoding      ,		lowercase_							:	Optional[Callable] = None    )   ->		Union[str, Any]:
       lowercase__					:  Optional[int]          =							pickle.dumps(lowercase_    )
       self.assertIsNotNone(lowercase_    )

       lowercase__					:  List[Any]          =							pickle.loads(lowercase_    )

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast      ,		be_original.is_fast    )

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
           self.assertIsNotNone(be_restored.encodings    )
       else:
           self.assertIsNone(be_restored.encodings    )

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()      ,		be_restored.values()    ):
           if equal_op:
               self.assertTrue(equal_op(lowercase_      ,		lowercase_    )    )
           else:
               self.assertEqual(lowercase_      ,		lowercase_    )






   @slow
   def  __UpperCamelCase	( self							:	Dict    )   ->		List[Any]:
       self.check_tokenizer_from_pretrained(lowercase_    )






   def  __UpperCamelCase	( self							:	Optional[Any]    )   ->		Tuple:
       self.assertEqual(TensorType("tf"    )      ,		TensorType.TENSORFLOW    )
       self.assertEqual(TensorType("pt"    )      ,		TensorType.PYTORCH    )
       self.assertEqual(TensorType("np"    )      ,		TensorType.NUMPY    )






   @require_tokenizers
   def  __UpperCamelCase	( self							:	int    )   ->		Union[str, Any]:
       import numpy as np

       lowercase__					:  Dict          =							BertTokenizer.from_pretrained("bert-base-cased"    )
       lowercase__					:  Union[str, Any]          =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

       # Python no tensor
       with self.subTest("BatchEncoding (Python, return_tensors=None)"    ):
           self.assert_dump_and_restore(tokenizer_p("Small example to encode"    )    )

       with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"    ):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"      ,		return_tensors=TensorType.NUMPY    )      ,		np.array_equal    )

       with self.subTest("BatchEncoding (Rust, return_tensors=None)"    ):
           self.assert_dump_and_restore(tokenizer_r("Small example to encode"    )    )

       with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"    ):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"      ,		return_tensors=TensorType.NUMPY    )      ,		np.array_equal    )






   @require_tf
   @require_tokenizers
   def  __UpperCamelCase	( self							:	List[str]    )   ->		Optional[Any]:
       import tensorflow as tf

       def tf_array_equals(lowercase_							:	List[Any]      ,		lowercase_							:	Any    ):
           return tf.reduce_all(tf.equal(lowercase_      ,		lowercase_    )    )

       lowercase__					:  int          =							BertTokenizer.from_pretrained("bert-base-cased"    )
       lowercase__					:  Any          =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

       with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"    ):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"      ,		return_tensors=TensorType.TENSORFLOW    )      ,		lowercase_    )

       with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"    ):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"      ,		return_tensors=TensorType.TENSORFLOW    )      ,		lowercase_    )






   @require_torch
   @require_tokenizers
   def  __UpperCamelCase	( self							:	List[Any]    )   ->		Dict:
       import torch

       lowercase__					:  List[str]          =							BertTokenizer.from_pretrained("bert-base-cased"    )
       lowercase__					:  Tuple          =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

       with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"    ):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"      ,		return_tensors=TensorType.PYTORCH    )      ,		torch.equal    )

       with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"    ):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"      ,		return_tensors=TensorType.PYTORCH    )      ,		torch.equal    )






   @require_tokenizers
   def  __UpperCamelCase	( self							:	Dict    )   ->		Any:
       lowercase__					:  Any          =							BertTokenizer.from_pretrained("bert-base-cased"    )
       lowercase__					:  Optional[int]          =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

       with self.subTest("Python Tokenizer"    ):
           self.assertFalse(tokenizer_p("Small example to_encode"    ).is_fast    )

       with self.subTest("Rust Tokenizer"    ):
           self.assertTrue(tokenizer_r("Small example to_encode"    ).is_fast    )






   @require_tokenizers
   def  __UpperCamelCase	( self							:	Tuple    )   ->		Tuple:
       lowercase__					:  Dict          =							BertTokenizerFast.from_pretrained("bert-base-cased"    )
       lowercase__					:  Any          =							tokenizer_r(["Test", "\xad", "test"]      ,		is_split_into_words=lowercase_    )

       self.assertEqual(encoded.word_to_tokens(0    )      ,		TokenSpan(start=1      ,		end=2    )    )
       self.assertEqual(encoded.word_to_tokens(1    )      ,		lowercase_    )
       self.assertEqual(encoded.word_to_tokens(2    )      ,		TokenSpan(start=2      ,		end=3    )    )






   def  __UpperCamelCase	( self							:	Dict    )   ->		Optional[int]:
       lowercase__					:  Dict          =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
       lowercase__					:  Optional[int]          =							batch.convert_to_tensors(tensor_type="np"    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(2, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__					:  Any          =							batch.convert_to_tensors(tensor_type="np"    )
       self.assertFalse(len(cs.err    )      ,		msg=F'''should have no warning, but got {cs.err}'''    )

       lowercase__					:  Optional[Any]          =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
       lowercase__					:  Optional[int]          =							batch.convert_to_tensors(tensor_type="np"      ,		prepend_batch_axis=lowercase_    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(1, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(1,)    )






   @require_torch
   def  __UpperCamelCase	( self							:	Any    )   ->		List[str]:
       lowercase__					:  str          =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
       lowercase__					:  Optional[int]          =							batch.convert_to_tensors(tensor_type="pt"    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(2, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__					:  List[str]          =							batch.convert_to_tensors(tensor_type="pt"    )
       self.assertFalse(len(cs.err    )      ,		msg=F'''should have no warning, but got {cs.err}'''    )

       lowercase__					:  Dict          =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
       lowercase__					:  Optional[Any]          =							batch.convert_to_tensors(tensor_type="pt"      ,		prepend_batch_axis=lowercase_    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(1, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(1,)    )






   @require_tf
   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		Union[str, Any]:
       lowercase__					:  Union[str, Any]          =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
       lowercase__					:  Tuple          =							batch.convert_to_tensors(tensor_type="tf"    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(2, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__					:  Dict          =							batch.convert_to_tensors(tensor_type="tf"    )
       self.assertFalse(len(cs.err    )      ,		msg=F'''should have no warning, but got {cs.err}'''    )

       lowercase__					:  Union[str, Any]          =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
       lowercase__					:  List[Any]          =							batch.convert_to_tensors(tensor_type="tf"      ,		prepend_batch_axis=lowercase_    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(1, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(1,)    )






   @require_flax
   def  __UpperCamelCase	( self							:	Any    )   ->		Optional[Any]:
       lowercase__					:  Union[str, Any]          =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
       lowercase__					:  Optional[int]          =							batch.convert_to_tensors(tensor_type="jax"    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(2, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__					:  str          =							batch.convert_to_tensors(tensor_type="jax"    )
       self.assertFalse(len(cs.err    )      ,		msg=F'''should have no warning, but got {cs.err}'''    )

       lowercase__					:  Union[str, Any]          =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
       lowercase__					:  Optional[int]          =							batch.convert_to_tensors(tensor_type="jax"      ,		prepend_batch_axis=lowercase_    )
       self.assertEqual(tensor_batch["inputs"].shape      ,		(1, 3)    )
       self.assertEqual(tensor_batch["labels"].shape      ,		(1,)    )






   def  __UpperCamelCase	( self							:	Optional[int]    )   ->		str:
       lowercase__					:  List[str]          =							[{"input_ids": np.array([0, 1, 2]    )}, {"input_ids": np.array([0, 1, 2, 3]    )}]
       lowercase__					:  Optional[int]          =							BertTokenizer.from_pretrained("bert-base-cased"    )

       lowercase__					:  Optional[int]          =							tokenizer.pad(lowercase_      ,		padding=lowercase_    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		np.ndarray    )    )
       self.assertEqual(batch["input_ids"].tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowercase__					:  Optional[Any]          =							tokenizer.pad(lowercase_      ,		padding=lowercase_      ,		return_tensors="np"    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		np.ndarray    )    )
       self.assertEqual(batch["input_ids"].tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )






   @require_torch
   def  __UpperCamelCase	( self							:	Optional[Any]    )   ->		Union[str, Any]:
       import torch

       lowercase__					:  List[Any]          =							[{"input_ids": torch.tensor([0, 1, 2]    )}, {"input_ids": torch.tensor([0, 1, 2, 3]    )}]
       lowercase__					:  Optional[int]          =							BertTokenizer.from_pretrained("bert-base-cased"    )

       lowercase__					:  List[str]          =							tokenizer.pad(lowercase_      ,		padding=lowercase_    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		torch.Tensor    )    )
       self.assertEqual(batch["input_ids"].tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowercase__					:  Optional[Any]          =							tokenizer.pad(lowercase_      ,		padding=lowercase_      ,		return_tensors="pt"    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		torch.Tensor    )    )
       self.assertEqual(batch["input_ids"].tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )






   @require_tf
   def  __UpperCamelCase	( self							:	Union[str, Any]    )   ->		str:
       import tensorflow as tf

       lowercase__					:  Tuple          =							[{"input_ids": tf.constant([0, 1, 2]    )}, {"input_ids": tf.constant([0, 1, 2, 3]    )}]
       lowercase__					:  Union[str, Any]          =							BertTokenizer.from_pretrained("bert-base-cased"    )

       lowercase__					:  str          =							tokenizer.pad(lowercase_      ,		padding=lowercase_    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		tf.Tensor    )    )
       self.assertEqual(batch["input_ids"].numpy().tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowercase__					:  Union[str, Any]          =							tokenizer.pad(lowercase_      ,		padding=lowercase_      ,		return_tensors="tf"    )
       self.assertTrue(isinstance(batch["input_ids"]      ,		tf.Tensor    )    )
       self.assertEqual(batch["input_ids"].numpy().tolist()      ,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )






   @require_tokenizers
   def  __UpperCamelCase	( self							:	Any    )   ->		Dict:
       lowercase__					:  Tuple          =							Tokenizer(WordPiece(unk_token="[UNK]"    )    )
       PreTrainedTokenizerFast(tokenizer_object=lowercase_    )







   @require_tokenizers
   def  __UpperCamelCase	( self							:	int    )   ->		Tuple:
       lowercase__					:  Dict          =							Tokenizer(WordPiece(unk_token="[UNK]"    )    )
       with tempfile.TemporaryDirectory() as tmpdirname:
           bert_tokenizer.save(os.path.join(lowercase_      ,		"tokenizer.json"    )    )
           PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowercase_      ,		"tokenizer.json"    )    )




def      lowercase_							(     _lowerCamelCase				:			str):
    lowercase__					:  Union[str, Any]          =							0
    for plain_chr in plain_text:
        lowercase__					:  Union[str, Any]          =							(
            ord(_lowerCamelCase) + (hash_value << 6) + (hash_value << 16) - hash_value
        )
    return hash_value



