




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
    from tokenizers import Tokenizer
    from tokenizers.models import WordPiece


class        UpperCAmelCase__				(							unittest.TestCase				):



     '''simple docstring'''
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
            a			   =list(tokenizer_class.max_model_input_sizes.keys()	)
            for model_name in sa_models[:1]:
                   a			   =tokenizer_class.from_pretrained(_lowerCAmelCase	)
                   self.assertIsNotNone(_lowerCAmelCase	)
                   self.assertIsInstance(_lowerCAmelCase ,    _lowerCAmelCase	)
                   self.assertIsInstance(_lowerCAmelCase ,    _lowerCAmelCase	)

                   for special_tok in tokenizer.all_special_tokens:
                          self.assertIsInstance(_lowerCAmelCase ,    _lowerCAmelCase	)
                          a			   =tokenizer.convert_tokens_to_ids(_lowerCAmelCase	)
                          self.assertIsInstance(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    _lowerCAmelCase = None	):
            a			   =pickle.dumps(_lowerCAmelCase	)
            self.assertIsNotNone(_lowerCAmelCase	)

            a			   =pickle.loads(_lowerCAmelCase	)

            # Ensure is_fast is correctly restored
            self.assertEqual(be_restored.is_fast ,    be_original.is_fast	)

            # Ensure encodings are potentially correctly restored
            if be_original.is_fast:
                   self.assertIsNotNone(be_restored.encodings	)
            else:
                   self.assertIsNone(be_restored.encodings	)

            # Ensure the keys are the same
            for original_v, restored_v in zip(be_original.values() ,    be_restored.values()	):
                   if equal_op:
                          self.assertTrue(equal_op(_lowerCAmelCase ,    _lowerCAmelCase	)	)
                   else:
                          self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     @slow
     def        lowerCAmelCase__  (				self	):
            self.check_tokenizer_from_pretrained(_lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            self.assertEqual(TensorType("""tf"""	) ,    TensorType.TENSORFLOW	)
            self.assertEqual(TensorType("""pt"""	) ,    TensorType.PYTORCH	)
            self.assertEqual(TensorType("""np"""	) ,    TensorType.NUMPY	)
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            import numpy as np

            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)
            a			   =BertTokenizerFast.from_pretrained("""bert-base-cased"""	)

            # Python no tensor
            with self.subTest("""BatchEncoding (Python, return_tensors=None)"""	):
                   self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""	)	)

            with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_p("""Small example to encode""" ,    return_tensors=TensorType.NUMPY	) ,    np.array_equal	)

            with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""	):
                   self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""	)	)

            with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_r("""Small example to encode""" ,    return_tensors=TensorType.NUMPY	) ,    np.array_equal	)
     @require_tf
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            import tensorflow as tf

            def tf_array_equals(_lowerCAmelCase ,    _lowerCAmelCase	):
                   return tf.reduce_all(tf.equal(_lowerCAmelCase ,    _lowerCAmelCase	)	)

            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)
            a			   =BertTokenizerFast.from_pretrained("""bert-base-cased"""	)

            with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_p("""Small example to encode""" ,    return_tensors=TensorType.TENSORFLOW	) ,    _lowerCAmelCase	)

            with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_r("""Small example to encode""" ,    return_tensors=TensorType.TENSORFLOW	) ,    _lowerCAmelCase	)
     @require_torch
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            import torch

            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)
            a			   =BertTokenizerFast.from_pretrained("""bert-base-cased"""	)

            with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_p("""Small example to encode""" ,    return_tensors=TensorType.PYTORCH	) ,    torch.equal	)

            with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""	):
                   self.assert_dump_and_restore(
                       tokenizer_r("""Small example to encode""" ,    return_tensors=TensorType.PYTORCH	) ,    torch.equal	)
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)
            a			   =BertTokenizerFast.from_pretrained("""bert-base-cased"""	)

            with self.subTest("""Python Tokenizer"""	):
                   self.assertFalse(tokenizer_p("""Small example to_encode"""	).is_fast	)

            with self.subTest("""Rust Tokenizer"""	):
                   self.assertTrue(tokenizer_r("""Small example to_encode"""	).is_fast	)
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            a			   =BertTokenizerFast.from_pretrained("""bert-base-cased"""	)
            a			   =tokenizer_r(["""Test""", """\xad""", """test"""] ,    is_split_into_words=_lowerCAmelCase	)

            self.assertEqual(encoded.word_to_tokens(0	) ,    TokenSpan(start=1 ,    end=2	)	)
            self.assertEqual(encoded.word_to_tokens(1	) ,    _lowerCAmelCase	)
            self.assertEqual(encoded.word_to_tokens(2	) ,    TokenSpan(start=2 ,    end=3	)	)
     def        lowerCAmelCase__  (				self	):
            a			   =BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}	)
            a			   =batch.convert_to_tensors(tensor_type="""np"""	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (2, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (2,)	)
            # test converting the converted
            with CaptureStderr() as cs:
                   a			   =batch.convert_to_tensors(tensor_type="""np"""	)
            self.assertFalse(len(cs.err	) ,    msg=F'''should have no warning, but got {cs.err}'''	)

            a			   =BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}	)
            a			   =batch.convert_to_tensors(tensor_type="""np""" ,    prepend_batch_axis=_lowerCAmelCase	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (1, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (1,)	)
     @require_torch
     def        lowerCAmelCase__  (				self	):
            a			   =BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}	)
            a			   =batch.convert_to_tensors(tensor_type="""pt"""	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (2, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (2,)	)
            # test converting the converted
            with CaptureStderr() as cs:
                   a			   =batch.convert_to_tensors(tensor_type="""pt"""	)
            self.assertFalse(len(cs.err	) ,    msg=F'''should have no warning, but got {cs.err}'''	)

            a			   =BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}	)
            a			   =batch.convert_to_tensors(tensor_type="""pt""" ,    prepend_batch_axis=_lowerCAmelCase	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (1, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (1,)	)
     @require_tf
     def        lowerCAmelCase__  (				self	):
            a			   =BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}	)
            a			   =batch.convert_to_tensors(tensor_type="""tf"""	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (2, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (2,)	)
            # test converting the converted
            with CaptureStderr() as cs:
                   a			   =batch.convert_to_tensors(tensor_type="""tf"""	)
            self.assertFalse(len(cs.err	) ,    msg=F'''should have no warning, but got {cs.err}'''	)

            a			   =BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}	)
            a			   =batch.convert_to_tensors(tensor_type="""tf""" ,    prepend_batch_axis=_lowerCAmelCase	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (1, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (1,)	)
     @require_flax
     def        lowerCAmelCase__  (				self	):
            a			   =BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}	)
            a			   =batch.convert_to_tensors(tensor_type="""jax"""	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (2, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (2,)	)
            # test converting the converted
            with CaptureStderr() as cs:
                   a			   =batch.convert_to_tensors(tensor_type="""jax"""	)
            self.assertFalse(len(cs.err	) ,    msg=F'''should have no warning, but got {cs.err}'''	)

            a			   =BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}	)
            a			   =batch.convert_to_tensors(tensor_type="""jax""" ,    prepend_batch_axis=_lowerCAmelCase	)
            self.assertEqual(tensor_batch["""inputs"""].shape ,    (1, 3)	)
            self.assertEqual(tensor_batch["""labels"""].shape ,    (1,)	)
     def        lowerCAmelCase__  (				self	):
            a			   =[{"""input_ids""": np.array([0, 1, 2]	)}, {"""input_ids""": np.array([0, 1, 2, 3]	)}]
            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)

            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    np.ndarray	)	)
            self.assertEqual(batch["""input_ids"""].tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase ,    return_tensors="""np"""	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    np.ndarray	)	)
            self.assertEqual(batch["""input_ids"""].tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
     @require_torch
     def        lowerCAmelCase__  (				self	):
            import torch

            a			   =[{"""input_ids""": torch.tensor([0, 1, 2]	)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]	)}]
            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)

            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    torch.Tensor	)	)
            self.assertEqual(batch["""input_ids"""].tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase ,    return_tensors="""pt"""	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    torch.Tensor	)	)
            self.assertEqual(batch["""input_ids"""].tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
     @require_tf
     def        lowerCAmelCase__  (				self	):
            import tensorflow as tf

            a			   =[{"""input_ids""": tf.constant([0, 1, 2]	)}, {"""input_ids""": tf.constant([0, 1, 2, 3]	)}]
            a			   =BertTokenizer.from_pretrained("""bert-base-cased"""	)

            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    tf.Tensor	)	)
            self.assertEqual(batch["""input_ids"""].numpy().tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
            a			   =tokenizer.pad(_lowerCAmelCase ,    padding=_lowerCAmelCase ,    return_tensors="""tf"""	)
            self.assertTrue(isinstance(batch["""input_ids"""] ,    tf.Tensor	)	)
            self.assertEqual(batch["""input_ids"""].numpy().tolist() ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            a			   =Tokenizer(WordPiece(unk_token="""[UNK]"""	)	)
            PreTrainedTokenizerFast(tokenizer_object=_lowerCAmelCase	)






     @require_tokenizers
     def        lowerCAmelCase__  (				self	):
            a			   =Tokenizer(WordPiece(unk_token="""[UNK]"""	)	)
            with tempfile.TemporaryDirectory() as tmpdirname:
                   bert_tokenizer.save(os.path.join(_lowerCAmelCase ,    """tokenizer.json"""	)	)
                   PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowerCAmelCase ,    """tokenizer.json"""	)	)






from __future__ import annotations

from typing import Any


class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self ,    _lowerCAmelCase ,    _lowerCAmelCase	):  # noqa: A002
            a			   =item
            a			   =next



class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self	):
            a			   =None
            a			   =0
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
            a			   =Node(_lowerCAmelCase ,    self.head	)
            self.size += 1
     def        lowerCAmelCase__  (				self	):
            # Switched 'self.is_empty()' to 'self.head is None'
            # because mypy was considering the possibility that 'self.head'
            # can be None in below else part and giving error
            if self.head is None:
                   return None
            else:
                   a			   =self.head.item
                   a			   =self.head.next
                   self.size -= 1
                   return item
     def        lowerCAmelCase__  (				self	):
            return self.head is None
     def __str__(				self	):
            if self.is_empty():
                   return ""
            else:
                   a			   =self.head
                   a			   =""""""
                   a			   =[]
                   while iterate:
                          item_list.append(str(iterate.item	)	)
                          a			   =iterate.next

                   a			   =""" --> """.join(_lowerCAmelCase	)

                   return item_str






     def __len__(				self	):
            return self.size
