





import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase

@require_tokenizers
class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     __snake_case       :      List[Any]						      =     EsmTokenizer






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Tuple:




           '''simple docstring'''


           super().setUp()
           SCREAMING_SNAKE_CASE						=		tempfile.mkdtemp()
           # fmt: off
           SCREAMING_SNAKE_CASE						=		["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
           # fmt: on
           SCREAMING_SNAKE_CASE						=		os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES["""vocab_file"""]			)
           with open(self.vocab_file		,"""w"""		,encoding="""utf-8"""			) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]		,**lowerCamelCase__		:       int			)	->						List[PreTrainedTokenizerBase]:




           '''simple docstring'''


           return [self.get_tokenizer(**lowerCamelCase__			)]






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]		,**lowerCamelCase__		:       Optional[Any]			)	->						PreTrainedTokenizer:




           '''simple docstring'''


           return self.tokenizer_class.from_pretrained(self.tmpdirname		,**lowerCamelCase__			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tokenizer_class(self.vocab_file			)

           SCREAMING_SNAKE_CASE						=		tokenizer.tokenize("""LAGVS"""			)
           self.assertListEqual(lowerCamelCase__		,["""L""", """A""", """G""", """V""", """S"""]			)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__			)		,[4, 5, 6, 7, 8]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Union[str, Any]			)	->						Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tokenizer_class(self.vocab_file			)

           SCREAMING_SNAKE_CASE						=		"""LAGVS"""
           self.assertListEqual(tokenizer.encode(lowerCamelCase__			)		,[0, 4, 5, 6, 7, 8, 2]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tokenizer_class(self.vocab_file			)

           SCREAMING_SNAKE_CASE						=		["""LAGVS""", """WCB"""]
           SCREAMING_SNAKE_CASE						=		tokenizer(lowerCamelCase__		,padding=lowerCamelCase__			)["""input_ids"""]

           self.assertListEqual(lowerCamelCase__		,[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Dict			)	->						Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.tokenizer_class(self.vocab_file			)

           SCREAMING_SNAKE_CASE						=		["""LAGVS""", """WCB"""]
           SCREAMING_SNAKE_CASE						=		tokenizer(lowerCamelCase__		,padding=lowerCamelCase__			)["""input_ids"""]

           self.assertListEqual(lowerCamelCase__		,[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[Any]			)	->						Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		self.get_tokenizers(fast=lowerCamelCase__			)
           for tokenizer in tokenizers:
                 with self.subTest(F"""{tokenizer.__class__.__name__}"""			):
                       SCREAMING_SNAKE_CASE						=		"""<unk>"""
                       SCREAMING_SNAKE_CASE						=		"""<mask>"""

                       SCREAMING_SNAKE_CASE						=		tokenizer.tokenize(lowerCamelCase__			)
                       SCREAMING_SNAKE_CASE						=		tokenizer.tokenize(lowerCamelCase__			)

                       self.assertEqual(len(lowerCamelCase__			)		,1			)
                       self.assertEqual(len(lowerCamelCase__			)		,1			)
                       self.assertEqual(token_a[0]		,lowerCamelCase__			)
                       self.assertEqual(token_a[0]		,lowerCamelCase__			)









import unittest
from dataclasses import dataclass
from typing import Optional

from transformers.utils import ModelOutput

@dataclass
class 						UpperCamelCase__					( lowerCAmelCase_  ):



     '''simple docstring'''

     __snake_case       :      float
     __snake_case       :      Optional[float]						      =     None
     __snake_case       :      Optional[float]						      =     None

class 						UpperCamelCase__					( unittest.TestCase  ):



     '''simple docstring'''

     def 							SCREAMING_SNAKE_CASE__    (				self		:       Any			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30			)
           self.assertEqual(x.a		,30			)
           self.assertIsNone(x.b			)
           self.assertIsNone(x.c			)
           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x.d






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[Any]			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,b=10			)
           self.assertEqual(x[0]		,30			)
           self.assertEqual(x[1]		,10			)
           self.assertEqual(x[:2]		,(30, 10)			)
           self.assertEqual(x[:]		,(30, 10)			)

           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,c=10			)
           self.assertEqual(x[0]		,30			)
           self.assertEqual(x[1]		,10			)
           self.assertEqual(x[:2]		,(30, 10)			)
           self.assertEqual(x[:]		,(30, 10)			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       str			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,b=10			)
           self.assertEqual(x["""a"""]		,30			)
           self.assertEqual(x["""b"""]		,10			)
           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x["""c"""]

           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,c=10			)
           self.assertEqual(x["""a"""]		,30			)
           self.assertEqual(x["""c"""]		,10			)
           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x["""b"""]






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30			)
           self.assertEqual(list(x.keys()			)		,["""a"""]			)
           self.assertEqual(list(x.values()			)		,[30]			)
           self.assertEqual(list(x.items()			)		,[("""a""", 30)]			)
           self.assertEqual(list(lowerCamelCase__			)		,["""a"""]			)

           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,b=10			)
           self.assertEqual(list(x.keys()			)		,["""a""", """b"""]			)
           self.assertEqual(list(x.values()			)		,[30, 10]			)
           self.assertEqual(list(x.items()			)		,[("""a""", 30), ("""b""", 10)]			)
           self.assertEqual(list(lowerCamelCase__			)		,["""a""", """b"""]			)

           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30		,c=10			)
           self.assertEqual(list(x.keys()			)		,["""a""", """c"""]			)
           self.assertEqual(list(x.values()			)		,[30, 10]			)
           self.assertEqual(list(x.items()			)		,[("""a""", 30), ("""c""", 10)]			)
           self.assertEqual(list(lowerCamelCase__			)		,["""a""", """c"""]			)

           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x.update({"""d""": 20}			)
           with self.assertRaises(lowerCamelCase__			):
                 del x["a"]
           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x.pop("""a"""			)
           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		x.setdefault("""d"""		,32			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       int			)	->						List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30			)
           SCREAMING_SNAKE_CASE						=		10
           self.assertEqual(x.a		,10			)
           self.assertEqual(x["""a"""]		,10			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[str]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=30			)
           SCREAMING_SNAKE_CASE						=		10
           self.assertEqual(x.a		,10			)
           self.assertEqual(x["""a"""]		,10			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       List[Any]			)	->						Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest({"""a""": 30, """b""": 10}			)
           self.assertEqual(list(x.keys()			)		,["""a""", """b"""]			)
           self.assertEqual(x.a		,30			)
           self.assertEqual(x.b		,10			)






     def 							SCREAMING_SNAKE_CASE__    (				self		:       Optional[int]			)	->						Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE						=		ModelOutputTest([("""a""", 30), ("""b""", 10)]			)
           self.assertEqual(list(x.keys()			)		,["""a""", """b"""]			)
           self.assertEqual(x.a		,30			)
           self.assertEqual(x.b		,10			)

           with self.assertRaises(lowerCamelCase__			):
                 SCREAMING_SNAKE_CASE						=		ModelOutputTest([("""a""", 30), (10, 10)]			)

           SCREAMING_SNAKE_CASE						=		ModelOutputTest(a=(30, 30)			)
           self.assertEqual(list(x.keys()			)		,["""a"""]			)
           self.assertEqual(x.a		,(30, 30)			)


