





'''simple docstring'''







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class        __a   (lowerCamelCase							,		unittest.TestCase  ):
							__a       : Tuple          =							HerbertTokenizer
							__a       : int          =							HerbertTokenizerFast
							__a       : Tuple          =							True

							def      UpperCAmelCase__						(				self				:  List[str] )       ->							Optional[Any]:







													"""simple docstring"""
													super().setUp()

													# Use a simpler test file without japanese/chinese characters
													with open(F"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"""							,  encoding='''utf-8''' ) as f_data:
																			UpperCAmelCase_      :   List[Any]      =	f_data.read().replace('''\n\n'''							,  '''\n''' ).strip()

													UpperCAmelCase_      :   Optional[int]      =	[
													    '''<s>''',
													    '''</s>''',
													    '''l''',
													    '''o''',
													    '''w''',
													    '''e''',
													    '''r''',
													    '''s''',
													    '''t''',
													    '''i''',
													    '''d''',
													    '''n''',
													    '''w</w>''',
													    '''r</w>''',
													    '''t</w>''',
													    '''lo''',
													    '''low''',
													    '''er</w>''',
													    '''low</w>''',
													    '''lowest</w>''',
													    '''newer</w>''',
													    '''wider</w>''',
													    ''',</w>''',
													    '''<unk>''',
													]
													UpperCAmelCase_      :   Optional[int]      =	dict(zip(__magic_name__							,  range(len(__magic_name__ ) ) ) )
													UpperCAmelCase_      :   List[Any]      =	['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

													UpperCAmelCase_      :   Optional[int]      =	os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''vocab_file'''] )
													UpperCAmelCase_      :   List[Any]      =	os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''merges_file'''] )
													with open(self.vocab_file							,  '''w''' ) as fp:
																			fp.write(json.dumps(__magic_name__ ) )
													with open(self.merges_file							,  '''w''' ) as fp:
																			fp.write('''\n'''.join(__magic_name__ ) )

							def      UpperCAmelCase__						(				self				:  Optional[Any]							,  __magic_name__				:  Optional[Any] )       ->							int:







													"""simple docstring"""
													UpperCAmelCase_      :   str      =	'''lower newer'''
													UpperCAmelCase_      :   List[str]      =	'''lower newer'''
													return input_text, output_text

							def      UpperCAmelCase__						(				self				:  Union[str, Any] )       ->							List[str]:







													"""simple docstring"""
													UpperCAmelCase_      :   List[Any]      =	self.tokenizer_class(vocab_file=self.vocab_file							,  merges_file=self.merges_file )

													UpperCAmelCase_      :   List[str]      =	'''lower'''
													UpperCAmelCase_      :   List[str]      =	['''low''', '''er</w>''']
													UpperCAmelCase_      :   Optional[int]      =	tokenizer.tokenize(__magic_name__ )
													self.assertListEqual(__magic_name__							,  __magic_name__ )

													UpperCAmelCase_      :   Dict      =	tokens + ['''<unk>''']
													UpperCAmelCase_      :   List[str]      =	[16, 17, 23]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__ )							,  __magic_name__ )

							def      UpperCAmelCase__						(				self				:  int )       ->							int:







													"""simple docstring"""
													if not self.test_rust_tokenizer:
																			return

													UpperCAmelCase_      :   Dict      =	self.get_tokenizer()
													UpperCAmelCase_      :   Tuple      =	self.get_rust_tokenizer()

													UpperCAmelCase_      :   int      =	'''lower,newer'''

													UpperCAmelCase_      :   Any      =	tokenizer.tokenize(__magic_name__ )
													UpperCAmelCase_      :   List[str]      =	rust_tokenizer.tokenize(__magic_name__ )
													self.assertListEqual(__magic_name__							,  __magic_name__ )

													UpperCAmelCase_      :   Optional[int]      =	tokenizer.encode(__magic_name__							,  add_special_tokens=__magic_name__ )
													UpperCAmelCase_      :   List[str]      =	rust_tokenizer.encode(__magic_name__							,  add_special_tokens=__magic_name__ )
													self.assertListEqual(__magic_name__							,  __magic_name__ )

													UpperCAmelCase_      :   Any      =	self.get_rust_tokenizer()
													UpperCAmelCase_      :   str      =	tokenizer.encode(__magic_name__ )
													UpperCAmelCase_      :   Optional[int]      =	rust_tokenizer.encode(__magic_name__ )
													self.assertListEqual(__magic_name__							,  __magic_name__ )



							@slow
							def      UpperCAmelCase__						(				self				:  Dict )       ->							Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_      :   Dict      =	self.tokenizer_class.from_pretrained('''allegro/herbert-base-cased''' )

													UpperCAmelCase_      :   Any      =	tokenizer.encode('''konstruowanie sekwencji'''							,  add_special_tokens=__magic_name__ )
													UpperCAmelCase_      :   List[Any]      =	tokenizer.encode('''konstruowanie wielu sekwencji'''							,  add_special_tokens=__magic_name__ )

													UpperCAmelCase_      :   str      =	tokenizer.build_inputs_with_special_tokens(__magic_name__ )
													UpperCAmelCase_      :   Optional[Any]      =	tokenizer.build_inputs_with_special_tokens(__magic_name__							,  __magic_name__ )

													assert encoded_sentence == [0] + text + [2]
													assert encoded_pair == [0] + text + [2] + text_a + [2]












'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_  :							Any            =	{
    "configuration_dpr": ["DPR_PRETRAINED_CONFIG_ARCHIVE_MAP", "DPRConfig"],
    "tokenization_dpr": [
        "DPRContextEncoderTokenizer",
        "DPRQuestionEncoderTokenizer",
        "DPRReaderOutput",
        "DPRReaderTokenizer",
    ],
}


try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							str            =	[
					    "DPRContextEncoderTokenizerFast",
					    "DPRQuestionEncoderTokenizerFast",
					    "DPRReaderTokenizerFast",
					]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							Optional[int]            =	[
					    "DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPRContextEncoder",
					    "DPRPretrainedContextEncoder",
					    "DPRPreTrainedModel",
					    "DPRPretrainedQuestionEncoder",
					    "DPRPretrainedReader",
					    "DPRQuestionEncoder",
					    "DPRReader",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							Dict            =	[
					    "TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TFDPRContextEncoder",
					    "TFDPRPretrainedContextEncoder",
					    "TFDPRPretrainedQuestionEncoder",
					    "TFDPRPretrainedReader",
					    "TFDPRQuestionEncoder",
					    "TFDPRReader",
					]


if TYPE_CHECKING:
					from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
					from .tokenization_dpr import (
					    DPRContextEncoderTokenizer,
					    DPRQuestionEncoderTokenizer,
					    DPRReaderOutput,
					    DPRReaderTokenizer,
					)

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_dpr_fast import (
										    DPRContextEncoderTokenizerFast,
										    DPRQuestionEncoderTokenizerFast,
										    DPRReaderTokenizerFast,
										)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_dpr import (
										    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPRContextEncoder,
										    DPRPretrainedContextEncoder,
										    DPRPreTrainedModel,
										    DPRPretrainedQuestionEncoder,
										    DPRPretrainedReader,
										    DPRQuestionEncoder,
										    DPRReader,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_dpr import (
										    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFDPRContextEncoder,
										    TFDPRPretrainedContextEncoder,
										    TFDPRPretrainedQuestionEncoder,
										    TFDPRPretrainedReader,
										    TFDPRQuestionEncoder,
										    TFDPRReader,
										)

else:
					import sys

					snake_case_  :							List[str]            =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





