






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__snake_case     :     List[Any]       =  {
    """configuration_markuplm""": ["""MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MarkupLMConfig"""],
    """feature_extraction_markuplm""": ["""MarkupLMFeatureExtractor"""],
    """processing_markuplm""": ["""MarkupLMProcessor"""],
    """tokenization_markuplm""": ["""MarkupLMTokenizer"""],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case     :     Optional[Any]       =  ["""MarkupLMTokenizerFast"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case     :     Optional[Any]       =  [
					    """MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MarkupLMForQuestionAnswering""",
					    """MarkupLMForSequenceClassification""",
					    """MarkupLMForTokenClassification""",
					    """MarkupLMModel""",
					    """MarkupLMPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_markuplm import MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP, MarkupLMConfig
					from .feature_extraction_markuplm import MarkupLMFeatureExtractor
					from .processing_markuplm import MarkupLMProcessor
					from .tokenization_markuplm import MarkupLMTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_markuplm_fast import MarkupLMTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_markuplm import (
										    MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MarkupLMForQuestionAnswering,
										    MarkupLMForSequenceClassification,
										    MarkupLMForTokenClassification,
										    MarkupLMModel,
										    MarkupLMPreTrainedModel,
										)


else:
					import sys

					__snake_case     :     int       =  _LazyModule(__name__, globals()["""__file__"""], _import_structure)













from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       DPRContextEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRContextEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							str				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       DPRQuestionEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							List[Any]				=       DPRQuestionEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRReaderTokenizer
						_SCREAMING_SNAKE_CASE						:							int				=       DPRReaderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							List[str]				=       True






						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence ' * 4  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [[1_01] + text_a + [1_02] + text_a + [1_02] + text_a]
												lowerCAmelCase__         =    BatchEncoding({'input_ids': input_ids} )

												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [0]
												lowerCAmelCase__         =    DPRReaderOutput(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )

												lowerCAmelCase__    ,							lowerCAmelCase__         =    8, 9
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    tokenizer.decode_best_spans(_UpperCamelCase  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].start_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].end_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].doc_id  ,    0 )







						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [1_01] + text_a + [1_02] + text_a + [1_02] + text_a
												lowerCAmelCase__         =    tokenizer(questions=['question sequence']  ,    titles=['title sequence']  ,    texts=['text sequence'] )
												self.assertIn('input_ids'  ,    _UpperCamelCase )
												self.assertIn('attention_mask'  ,    _UpperCamelCase )
												self.assertListEqual(encoded_input['input_ids'][0]  ,    _UpperCamelCase )





