




'''simple docstring'''
import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase






@require_tokenizers
class 			SCREAMING_SNAKE_CASE(							unittest.TestCase	):
       snake_case_					:	List[Any]       =   EsmTokenizer

       def        snake_case__     (       self	)       ->   Tuple:

           """simple docstring"""



           super().setUp()
           __lowercase     							=		tempfile.mkdtemp()
           # fmt: off
           __lowercase     							=		["""<cls>""", """<pad>""", """<eos>""", """<unk>""", """L""", """A""", """G""", """V""", """S""", """E""", """R""", """T""", """I""", """D""", """P""", """K""", """Q""", """N""", """F""", """Y""", """M""", """H""", """W""", """C""", """X""", """B""", """U""", """Z""", """O""", """.""", """-""", """<null_1>""", """<mask>"""]  # noqa: E501
           # fmt: on
           __lowercase     							=		os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES["""vocab_file"""]	)
           with open(self.vocab_file   ,						"""w"""   ,						encoding="""utf-8"""	) as vocab_writer:
               vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)

       def        snake_case__     (       self   ,						**lowerCamelCase__	)       ->   Tuple:

           """simple docstring"""



           return [self.get_tokenizer(**__A	)]

       def        snake_case__     (       self   ,						**lowerCamelCase__	)       ->   Union[str, Any]:

           """simple docstring"""



           return self.tokenizer_class.from_pretrained(self.tmpdirname   ,						**__A	)

       def        snake_case__     (       self	)       ->   Tuple:

           """simple docstring"""



           __lowercase     							=		self.tokenizer_class(self.vocab_file	)

           __lowercase     							=		tokenizer.tokenize("""LAGVS"""	)
           self.assertListEqual(__A   ,						["""L""", """A""", """G""", """V""", """S"""]	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)   ,						[4, 5, 6, 7, 8]	)

       def        snake_case__     (       self	)       ->   List[Any]:

           """simple docstring"""



           __lowercase     							=		self.tokenizer_class(self.vocab_file	)

           __lowercase     							=		"""LAGVS"""
           self.assertListEqual(tokenizer.encode(__A	)   ,						[0, 4, 5, 6, 7, 8, 2]	)

       def        snake_case__     (       self	)       ->   List[str]:

           """simple docstring"""



           __lowercase     							=		self.tokenizer_class(self.vocab_file	)

           __lowercase     							=		["""LAGVS""", """WCB"""]
           __lowercase     							=		tokenizer(__A   ,						padding=__A	)["""input_ids"""]

           self.assertListEqual(__A   ,						[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]	)

       def        snake_case__     (       self	)       ->   Optional[Any]:

           """simple docstring"""



           __lowercase     							=		self.tokenizer_class(self.vocab_file	)

           __lowercase     							=		["""LAGVS""", """WCB"""]
           __lowercase     							=		tokenizer(__A   ,						padding=__A	)["""input_ids"""]

           self.assertListEqual(__A   ,						[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]	)




       def        snake_case__     (       self	)       ->   List[Any]:

           """simple docstring"""



           __lowercase     							=		self.get_tokenizers(fast=__A	)
           for tokenizer in tokenizers:
               with self.subTest(F'{tokenizer.__class__.__name__}'	):
                   __lowercase     							=		"""<unk>"""
                   __lowercase     							=		"""<mask>"""

                   __lowercase     							=		tokenizer.tokenize(__A	)
                   __lowercase     							=		tokenizer.tokenize(__A	)

                   self.assertEqual(len(__A	)   ,						1	)
                   self.assertEqual(len(__A	)   ,						1	)
                   self.assertEqual(token_a[0]   ,						__A	)
                   self.assertEqual(token_a[0]   ,						__A	)












"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a							:						Union[str, Any]   							=			logging.get_logger(__name__)

__a							:						Optional[Any]   							=			{
    'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/config.json',
    'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/config.json',
    'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/config.json',
    'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/config.json',
    # See all Splinter models at https://huggingface.co/models?filter=splinter
}







class 							_SCREAMING_SNAKE_CASE    (	__snake_case				):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE     ='splinter'




							def __init__( self:  Dict					,		__A:  List[Any]=30522					,		__A:  str=768					,		__A:  int=12					,		__A:  str=12					,		__A:  Tuple=3072					,		__A:  Dict="gelu"					,		__A:  Optional[int]=0.1					,		__A:  List[str]=0.1					,		__A:  str=512					,		__A:  List[Any]=2					,		__A:  Any=0.0_2					,		__A:  Any=1e-12					,		__A:  Tuple=True					,		__A:  List[Any]=0					,		__A:  List[str]=104					,		**__A:  Union[str, Any]					,		):

												'''simple docstring'''

												super().__init__(pad_token_id=__A					,		**__A     )

												a__ 							=	vocab_size
												a__ 							=	max_position_embeddings
												a__ 							=	hidden_size
												a__ 							=	num_hidden_layers
												a__ 							=	num_attention_heads
												a__ 							=	intermediate_size
												a__ 							=	hidden_act
												a__ 							=	hidden_dropout_prob
												a__ 							=	attention_probs_dropout_prob
												a__ 							=	initializer_range
												a__ 							=	type_vocab_size
												a__ 							=	layer_norm_eps
												a__ 							=	use_cache
												a__ 							=	question_token_id


