






"""simple docstring"""







import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase







@require_tokenizers
class 							_SCREAMING_SNAKE_CASE    (	unittest.TestCase				):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE     =EsmTokenizer




							def 			lowercase		( self:  Union[str, Any]     ):

												'''simple docstring'''

												super().setUp()
												a__ 							=	tempfile.mkdtemp()
												# fmt: off
												a__ 							=	["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
												# fmt: on
												a__ 							=	os.path.join(self.tmpdirname					,		VOCAB_FILES_NAMES['''vocab_file''']     )
												with open(self.vocab_file					,		'''w'''					,		encoding='''utf-8'''     ) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )




							def 			lowercase		( self:  str					,		**__A:  Union[str, Any]     ):

												'''simple docstring'''

												return [self.get_tokenizer(**__A     )]




							def 			lowercase		( self:  Optional[Any]					,		**__A:  List[str]     ):

												'''simple docstring'''

												return self.tokenizer_class.from_pretrained(self.tmpdirname					,		**__A     )




							def 			lowercase		( self:  Optional[Any]     ):

												'''simple docstring'''

												a__ 							=	self.tokenizer_class(self.vocab_file     )

												a__ 							=	tokenizer.tokenize('''LAGVS'''     )
												self.assertListEqual(__A					,		['''L''', '''A''', '''G''', '''V''', '''S''']     )
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__A     )					,		[4, 5, 6, 7, 8]     )




							def 			lowercase		( self:  Tuple     ):

												'''simple docstring'''

												a__ 							=	self.tokenizer_class(self.vocab_file     )

												a__ 							=	'''LAGVS'''
												self.assertListEqual(tokenizer.encode(__A     )					,		[0, 4, 5, 6, 7, 8, 2]     )




							def 			lowercase		( self:  Union[str, Any]     ):

												'''simple docstring'''

												a__ 							=	self.tokenizer_class(self.vocab_file     )

												a__ 							=	['''LAGVS''', '''WCB''']
												a__ 							=	tokenizer(__A					,		padding=__A     )['''input_ids''']

												self.assertListEqual(__A					,		[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]     )




							def 			lowercase		( self:  Union[str, Any]     ):

												'''simple docstring'''

												a__ 							=	self.tokenizer_class(self.vocab_file     )

												a__ 							=	['''LAGVS''', '''WCB''']
												a__ 							=	tokenizer(__A					,		padding=__A     )['''input_ids''']

												self.assertListEqual(__A					,		[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]     )





							def 			lowercase		( self:  List[str]     ):

												'''simple docstring'''

												a__ 							=	self.get_tokenizers(fast=__A     )
												for tokenizer in tokenizers:
																	with self.subTest(F'{tokenizer.__class__.__name__}'     ):
																						a__ 							=	'''<unk>'''
																						a__ 							=	'''<mask>'''

																						a__ 							=	tokenizer.tokenize(__A     )
																						a__ 							=	tokenizer.tokenize(__A     )

																						self.assertEqual(len(__A     )					,		1     )
																						self.assertEqual(len(__A     )					,		1     )
																						self.assertEqual(token_a[0]					,		__A     )
																						self.assertEqual(token_a[0]					,		__A     )










"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__a							:						int   							=			{
    'configuration_markuplm': ['MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MarkupLMConfig'],
    'feature_extraction_markuplm': ['MarkupLMFeatureExtractor'],
    'processing_markuplm': ['MarkupLMProcessor'],
    'tokenization_markuplm': ['MarkupLMTokenizer'],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a							:						List[Any]   							=			['MarkupLMTokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a							:						List[Any]   							=			[
	    'MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'MarkupLMForQuestionAnswering',
	    'MarkupLMForSequenceClassification',
	    'MarkupLMForTokenClassification',
	    'MarkupLMModel',
	    'MarkupLMPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_markuplm import MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP, MarkupLMConfig
	from .feature_extraction_markuplm import MarkupLMFeatureExtractor
	from .processing_markuplm import MarkupLMProcessor
	from .tokenization_markuplm import MarkupLMTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_markuplm_fast import MarkupLMTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_markuplm import (
		    MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MarkupLMForQuestionAnswering,
		    MarkupLMForSequenceClassification,
		    MarkupLMForTokenClassification,
		    MarkupLMModel,
		    MarkupLMPreTrainedModel,
		)


else:
	import sys

	__a							:						Tuple   							=			_LazyModule(__name__, globals()['__file__'], _import_structure)


