






"""simple docstring"""



from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   int											=						DPRContextEncoderTokenizer
    __lowercase    :   Any											=						DPRContextEncoderTokenizerFast
    __lowercase    :   Dict											=						True



@require_tokenizers
class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[int]											=						DPRQuestionEncoderTokenizer
    __lowercase    :   int											=						DPRQuestionEncoderTokenizerFast
    __lowercase    :   Tuple											=						True



@require_tokenizers
class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Tuple											=						DPRReaderTokenizer
    __lowercase    :   List[str]											=						DPRReaderTokenizerFast
    __lowercase    :   str											=						True


    @slow
    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.tokenizer_class.from_pretrained("""bert-base-uncased"""			)

     UpperCAmelCase__ 						=				tokenizer.encode("""question sequence"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.encode("""title sequence"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.encode("""text sequence """ * 4   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				[[101] + text_a + [102] + text_a + [102] + text_a]
     UpperCAmelCase__ 						=				BatchEncoding({"""input_ids""": input_ids}			)

     UpperCAmelCase__ 						=				[[0] * len(input_ids[0]			)]
     UpperCAmelCase__ 						=				[[0] * len(input_ids[0]			)]
     UpperCAmelCase__ 						=				[0]
     UpperCAmelCase__ 						=				DPRReaderOutput(__lowercase   ,			__lowercase   ,			__lowercase			)

     UpperCAmelCase__						,	UpperCAmelCase__ 						=				8, 9
     UpperCAmelCase__ 						=				10
     UpperCAmelCase__ 						=				10
     UpperCAmelCase__ 						=				tokenizer.decode_best_spans(__lowercase   ,			__lowercase			)
     self.assertEqual(predicted_spans[0].start_index   ,			__lowercase			)
     self.assertEqual(predicted_spans[0].end_index   ,			__lowercase			)
     self.assertEqual(predicted_spans[0].doc_id   ,			0			)






    @slow
    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.tokenizer_class.from_pretrained("""bert-base-uncased"""			)

     UpperCAmelCase__ 						=				tokenizer.encode("""question sequence"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.encode("""title sequence"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.encode("""text sequence"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				[101] + text_a + [102] + text_a + [102] + text_a
     UpperCAmelCase__ 						=				tokenizer(questions=["""question sequence"""]   ,			titles=["""title sequence"""]   ,			texts=["""text sequence"""]			)
     self.assertIn("""input_ids"""   ,			__lowercase			)
     self.assertIn("""attention_mask"""   ,			__lowercase			)
     self.assertListEqual(encoded_input["""input_ids"""][0]   ,			__lowercase			)











"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


a  :						List[Any]					       =							logging.get_logger(__name__)

a  :						List[Any]					       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a  :						Any					       =							{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

a  :						str					       =							{
    '''microsoft/mpnet-base''': 512,
}

a  :						Union[str, Any]					       =							{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}


class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						VOCAB_FILES_NAMES
    __lowercase    :   Optional[int]											=						PRETRAINED_VOCAB_FILES_MAP
    __lowercase    :   Dict											=						PRETRAINED_INIT_CONFIGURATION
    __lowercase    :   Optional[Any]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowercase    :   Dict											=						MPNetTokenizer
    __lowercase    :   Dict											=						['input_ids', 'attention_mask']


    def __init__(      self   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=True   ,			__lowercase="<s>"   ,			__lowercase="</s>"   ,			__lowercase="</s>"   ,			__lowercase="<s>"   ,			__lowercase="[UNK]"   ,			__lowercase="<pad>"   ,			__lowercase="<mask>"   ,			__lowercase=True   ,			__lowercase=None   ,			**__lowercase   ,			):
     super().__init__(
         __lowercase   ,			tokenizer_file=__lowercase   ,			do_lower_case=__lowercase   ,			bos_token=__lowercase   ,			eos_token=__lowercase   ,			sep_token=__lowercase   ,			cls_token=__lowercase   ,			unk_token=__lowercase   ,			pad_token=__lowercase   ,			mask_token=__lowercase   ,			tokenize_chinese_chars=__lowercase   ,			strip_accents=__lowercase   ,			**__lowercase   ,			)

     UpperCAmelCase__ 						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         pre_tok_state.get("""lowercase"""   ,			__lowercase			) != do_lower_case
         or pre_tok_state.get("""strip_accents"""   ,			__lowercase			) != strip_accents
     ):
      UpperCAmelCase__ 						=				getattr(__lowercase   ,			pre_tok_state.pop("""type"""			)			)
      UpperCAmelCase__ 						=				do_lower_case
      UpperCAmelCase__ 						=				strip_accents
      UpperCAmelCase__ 						=				pre_tok_class(**__lowercase			)

     UpperCAmelCase__ 						=				do_lower_case


    @property
    def 		A__      (      self			):
     if self._mask_token is None:
      if self.verbose:
       logger.error("""Using mask_token, but it is not set yet."""			)
      return None
     return str(self._mask_token			)


    @mask_token.setter
    def 		A__      (      self   ,			__lowercase			):
     UpperCAmelCase__ 						=				AddedToken(__lowercase   ,			lstrip=__lowercase   ,			rstrip=__lowercase			) if isinstance(__lowercase   ,			__lowercase			) else value
     UpperCAmelCase__ 						=				value


    def 		A__      (      self   ,			__lowercase   ,			__lowercase=None			):
     UpperCAmelCase__ 						=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
      return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				[self.sep_token_id]
     UpperCAmelCase__ 						=				[self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				self._tokenizer.model.save(__lowercase   ,			name=__lowercase			)
     return tuple(__lowercase			)



