





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece





class   lowercase__     (				unittest.TestCase	):







  def 						UpperCAmelCase__		(						self     ,    _lowercase    ):
       lowerCAmelCase_       :							Dict			 =		list(tokenizer_class.max_model_input_sizes.keys()    )
       for model_name in sa_models[:1]:
            lowerCAmelCase_       :							Optional[Any]			 =		tokenizer_class.from_pretrained(_lowerCAmelCase    )
            self.assertIsNotNone(_lowerCAmelCase    )
            self.assertIsInstance(_lowerCAmelCase     ,    _lowerCAmelCase    )
            self.assertIsInstance(_lowerCAmelCase     ,    _lowerCAmelCase    )

            for special_tok in tokenizer.all_special_tokens:
                 self.assertIsInstance(_lowerCAmelCase     ,    _lowerCAmelCase    )
                 lowerCAmelCase_       :							Optional[Any]			 =		tokenizer.convert_tokens_to_ids(_lowerCAmelCase    )
                 self.assertIsInstance(_lowerCAmelCase     ,    _lowerCAmelCase    )







  def 						UpperCAmelCase__		(						self     ,    _lowercase     ,    _lowercase = None    ):
       lowerCAmelCase_       :							Dict			 =		pickle.dumps(_lowerCAmelCase    )
       self.assertIsNotNone(_lowerCAmelCase    )

       lowerCAmelCase_       :							List[str]			 =		pickle.loads(_lowerCAmelCase    )

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast     ,    be_original.is_fast    )

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
            self.assertIsNotNone(be_restored.encodings    )
       else:
            self.assertIsNone(be_restored.encodings    )

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()     ,    be_restored.values()    ):
            if equal_op:
                 self.assertTrue(equal_op(_lowerCAmelCase     ,    _lowerCAmelCase    )    )
            else:
                 self.assertEqual(_lowerCAmelCase     ,    _lowerCAmelCase    )







  @slow
  def 						UpperCAmelCase__		(						self    ):
       self.check_tokenizer_from_pretrained(_lowerCAmelCase    )







  def 						UpperCAmelCase__		(						self    ):
       self.assertEqual(TensorType("""tf"""    )     ,    TensorType.TENSORFLOW    )
       self.assertEqual(TensorType("""pt"""    )     ,    TensorType.PYTORCH    )
       self.assertEqual(TensorType("""np"""    )     ,    TensorType.NUMPY    )







  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       import numpy as np

       lowerCAmelCase_       :							Union[str, Any]			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )
       lowerCAmelCase_       :							Tuple			 =		BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

       # Python no tensor
       with self.subTest("""BatchEncoding (Python, return_tensors=None)"""    ):
            self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""    )    )

       with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""    ):
            self.assert_dump_and_restore(
                tokenizer_p("""Small example to encode"""     ,    return_tensors=TensorType.NUMPY    )     ,    np.array_equal    )

       with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""    ):
            self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""    )    )

       with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""    ):
            self.assert_dump_and_restore(
                tokenizer_r("""Small example to encode"""     ,    return_tensors=TensorType.NUMPY    )     ,    np.array_equal    )







  @require_tf
  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       import tensorflow as tf

       def tf_array_equals(_lowercase     ,    _lowercase    ):
            return tf.reduce_all(tf.equal(_lowerCAmelCase     ,    _lowerCAmelCase    )    )

       lowerCAmelCase_       :							Union[str, Any]			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )
       lowerCAmelCase_       :							List[str]			 =		BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

       with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""    ):
            self.assert_dump_and_restore(
                tokenizer_p("""Small example to encode"""     ,    return_tensors=TensorType.TENSORFLOW    )     ,    _lowerCAmelCase    )

       with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""    ):
            self.assert_dump_and_restore(
                tokenizer_r("""Small example to encode"""     ,    return_tensors=TensorType.TENSORFLOW    )     ,    _lowerCAmelCase    )







  @require_torch
  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       import torch

       lowerCAmelCase_       :							int			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )
       lowerCAmelCase_       :							List[str]			 =		BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

       with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""    ):
            self.assert_dump_and_restore(
                tokenizer_p("""Small example to encode"""     ,    return_tensors=TensorType.PYTORCH    )     ,    torch.equal    )

       with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""    ):
            self.assert_dump_and_restore(
                tokenizer_r("""Small example to encode"""     ,    return_tensors=TensorType.PYTORCH    )     ,    torch.equal    )







  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							Union[str, Any]			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )
       lowerCAmelCase_       :							List[Any]			 =		BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

       with self.subTest("""Python Tokenizer"""    ):
            self.assertFalse(tokenizer_p("""Small example to_encode"""    ).is_fast    )

       with self.subTest("""Rust Tokenizer"""    ):
            self.assertTrue(tokenizer_r("""Small example to_encode"""    ).is_fast    )







  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							List[str]			 =		BertTokenizerFast.from_pretrained("""bert-base-cased"""    )
       lowerCAmelCase_       :							int			 =		tokenizer_r(["""Test""", """\xad""", """test"""]     ,    is_split_into_words=_lowerCAmelCase    )

       self.assertEqual(encoded.word_to_tokens(0    )     ,    TokenSpan(start=1     ,    end=2    )    )
       self.assertEqual(encoded.word_to_tokens(1    )     ,    _lowerCAmelCase    )
       self.assertEqual(encoded.word_to_tokens(2    )     ,    TokenSpan(start=2     ,    end=3    )    )







  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							str			 =		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
       lowerCAmelCase_       :							Tuple			 =		batch.convert_to_tensors(tensor_type="""np"""    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (2, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
            lowerCAmelCase_       :							int			 =		batch.convert_to_tensors(tensor_type="""np"""    )
       self.assertFalse(len(cs.err    )     ,    msg=F'should have no warning, but got {cs.err}'    )

       lowerCAmelCase_       :							Any			 =		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
       lowerCAmelCase_       :							Any			 =		batch.convert_to_tensors(tensor_type="""np"""     ,    prepend_batch_axis=_lowerCAmelCase    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (1, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (1,)    )







  @require_torch
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							Dict			 =		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
       lowerCAmelCase_       :							Optional[int]			 =		batch.convert_to_tensors(tensor_type="""pt"""    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (2, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
            lowerCAmelCase_       :							Optional[int]			 =		batch.convert_to_tensors(tensor_type="""pt"""    )
       self.assertFalse(len(cs.err    )     ,    msg=F'should have no warning, but got {cs.err}'    )

       lowerCAmelCase_       :							Optional[int]			 =		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
       lowerCAmelCase_       :							Optional[Any]			 =		batch.convert_to_tensors(tensor_type="""pt"""     ,    prepend_batch_axis=_lowerCAmelCase    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (1, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (1,)    )







  @require_tf
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							List[Any]			 =		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
       lowerCAmelCase_       :							Union[str, Any]			 =		batch.convert_to_tensors(tensor_type="""tf"""    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (2, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
            lowerCAmelCase_       :							Union[str, Any]			 =		batch.convert_to_tensors(tensor_type="""tf"""    )
       self.assertFalse(len(cs.err    )     ,    msg=F'should have no warning, but got {cs.err}'    )

       lowerCAmelCase_       :							Tuple			 =		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
       lowerCAmelCase_       :							Optional[int]			 =		batch.convert_to_tensors(tensor_type="""tf"""     ,    prepend_batch_axis=_lowerCAmelCase    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (1, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (1,)    )







  @require_flax
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							Dict			 =		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
       lowerCAmelCase_       :							Union[str, Any]			 =		batch.convert_to_tensors(tensor_type="""jax"""    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (2, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (2,)    )
       # test converting the converted
       with CaptureStderr() as cs:
            lowerCAmelCase_       :							List[Any]			 =		batch.convert_to_tensors(tensor_type="""jax"""    )
       self.assertFalse(len(cs.err    )     ,    msg=F'should have no warning, but got {cs.err}'    )

       lowerCAmelCase_       :							Union[str, Any]			 =		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
       lowerCAmelCase_       :							int			 =		batch.convert_to_tensors(tensor_type="""jax"""     ,    prepend_batch_axis=_lowerCAmelCase    )
       self.assertEqual(tensor_batch["""inputs"""].shape     ,    (1, 3)    )
       self.assertEqual(tensor_batch["""labels"""].shape     ,    (1,)    )







  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							List[str]			 =		[{"""input_ids""": np.array([0, 1, 2]    )}, {"""input_ids""": np.array([0, 1, 2, 3]    )}]
       lowerCAmelCase_       :							Dict			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )

       lowerCAmelCase_       :							Optional[Any]			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    np.ndarray    )    )
       self.assertEqual(batch["""input_ids"""].tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowerCAmelCase_       :							Dict			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase     ,    return_tensors="""np"""    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    np.ndarray    )    )
       self.assertEqual(batch["""input_ids"""].tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







  @require_torch
  def 						UpperCAmelCase__		(						self    ):
       import torch

       lowerCAmelCase_       :							Any			 =		[{"""input_ids""": torch.tensor([0, 1, 2]    )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]    )}]
       lowerCAmelCase_       :							Dict			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )

       lowerCAmelCase_       :							Optional[Any]			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    torch.Tensor    )    )
       self.assertEqual(batch["""input_ids"""].tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowerCAmelCase_       :							List[Any]			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase     ,    return_tensors="""pt"""    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    torch.Tensor    )    )
       self.assertEqual(batch["""input_ids"""].tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







  @require_tf
  def 						UpperCAmelCase__		(						self    ):
       import tensorflow as tf

       lowerCAmelCase_       :							int			 =		[{"""input_ids""": tf.constant([0, 1, 2]    )}, {"""input_ids""": tf.constant([0, 1, 2, 3]    )}]
       lowerCAmelCase_       :							Dict			 =		BertTokenizer.from_pretrained("""bert-base-cased"""    )

       lowerCAmelCase_       :							Optional[int]			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    tf.Tensor    )    )
       self.assertEqual(batch["""input_ids"""].numpy().tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
       lowerCAmelCase_       :							Tuple			 =		tokenizer.pad(_lowerCAmelCase     ,    padding=_lowerCAmelCase     ,    return_tensors="""tf"""    )
       self.assertTrue(isinstance(batch["""input_ids"""]     ,    tf.Tensor    )    )
       self.assertEqual(batch["""input_ids"""].numpy().tolist()     ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )







  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							List[str]			 =		Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
       PreTrainedTokenizerFast(tokenizer_object=_lowerCAmelCase    )







  @require_tokenizers
  def 						UpperCAmelCase__		(						self    ):
       lowerCAmelCase_       :							str			 =		Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
       with tempfile.TemporaryDirectory() as tmpdirname:
            bert_tokenizer.save(os.path.join(_lowerCAmelCase     ,    """tokenizer.json"""    )    )
            PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowerCAmelCase     ,    """tokenizer.json"""    )    )






from __future__ import annotations

from typing import Any


class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self ,    _lowerCAmelCase ,    _lowerCAmelCase	):  # noqa: A002
            a			   =item
            a			   =next



class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self	):
            a			   =None
            a			   =0
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
            a			   =Node(_lowerCAmelCase ,    self.head	)
            self.size += 1
     def        lowerCAmelCase__  (				self	):
            # Switched 'self.is_empty()' to 'self.head is None'
            # because mypy was considering the possibility that 'self.head'
            # can be None in below else part and giving error
            if self.head is None:
                   return None
            else:
                   a			   =self.head.item
                   a			   =self.head.next
                   self.size -= 1
                   return item
     def        lowerCAmelCase__  (				self	):
            return self.head is None
     def __str__(				self	):
            if self.is_empty():
                   return ""
            else:
                   a			   =self.head
                   a			   =""""""
                   a			   =[]
                   while iterate:
                          item_list.append(str(iterate.item	)	)
                          a			   =iterate.next

                   a			   =""" --> """.join(_lowerCAmelCase	)

                   return item_str






     def __len__(				self	):
            return self.size
