
import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          LayoutLMForMaskedLM,
          LayoutLMForQuestionAnswering,
          LayoutLMForSequenceClassification,
          LayoutLMForTokenClassification,
          LayoutLMModel,
      )






class     a_       :



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase				,       _lowerCamelCase=13				,       _lowerCamelCase=7				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=True				,       _lowerCamelCase=99				,       _lowerCamelCase=32				,       _lowerCamelCase=5				,       _lowerCamelCase=4				,       _lowerCamelCase=37				,       _lowerCamelCase="gelu"				,       _lowerCamelCase=0.1				,       _lowerCamelCase=0.1				,       _lowerCamelCase=512				,       _lowerCamelCase=16				,       _lowerCamelCase=2				,       _lowerCamelCase=0.0_2				,       _lowerCamelCase=3				,       _lowerCamelCase=4				,       _lowerCamelCase=None				,       _lowerCamelCase=1000				,       )					->str:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = parent
           SCREAMING_SNAKE_CASE				:				List[Any]				      = batch_size
           SCREAMING_SNAKE_CASE				:				Any				      = seq_length
           SCREAMING_SNAKE_CASE				:				List[str]				      = is_training
           SCREAMING_SNAKE_CASE				:				Any				      = use_input_mask
           SCREAMING_SNAKE_CASE				:				Dict				      = use_token_type_ids
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = use_labels
           SCREAMING_SNAKE_CASE				:				int				      = vocab_size
           SCREAMING_SNAKE_CASE				:				List[Any]				      = hidden_size
           SCREAMING_SNAKE_CASE				:				List[str]				      = num_hidden_layers
           SCREAMING_SNAKE_CASE				:				str				      = num_attention_heads
           SCREAMING_SNAKE_CASE				:				str				      = intermediate_size
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = hidden_act
           SCREAMING_SNAKE_CASE				:				Tuple				      = hidden_dropout_prob
           SCREAMING_SNAKE_CASE				:				List[str]				      = attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE				:				List[Any]				      = max_position_embeddings
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = type_vocab_size
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = type_sequence_label_size
           SCREAMING_SNAKE_CASE				:				Tuple				      = initializer_range
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = num_labels
           SCREAMING_SNAKE_CASE				:				List[str]				      = num_choices
           SCREAMING_SNAKE_CASE				:				List[str]				      = scope
           SCREAMING_SNAKE_CASE				:				Dict				      = range_bbox






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = ids_tensor([self.batch_size, self.seq_length]				,       self.vocab_size   )

           SCREAMING_SNAKE_CASE				:				List[str]				      = ids_tensor([self.batch_size, self.seq_length, 4]				,       self.range_bbox   )
           # Ensure that bbox is legal
           for i in range(bbox.shape[0]   ):
               for j in range(bbox.shape[1]   ):
                   if bbox[i, j, 3] < bbox[i, j, 1]:
                       SCREAMING_SNAKE_CASE				:				List[Any]				      = bbox[i, j, 3]
                       SCREAMING_SNAKE_CASE				:				Optional[Any]				      = bbox[i, j, 1]
                       SCREAMING_SNAKE_CASE				:				Tuple				      = t
                   if bbox[i, j, 2] < bbox[i, j, 0]:
                       SCREAMING_SNAKE_CASE				:				str				      = bbox[i, j, 2]
                       SCREAMING_SNAKE_CASE				:				List[Any]				      = bbox[i, j, 0]
                       SCREAMING_SNAKE_CASE				:				Any				      = t

           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = None
           if self.use_input_mask:
               SCREAMING_SNAKE_CASE				:				str				      = ids_tensor([self.batch_size, self.seq_length]				,       vocab_size=2   )

           SCREAMING_SNAKE_CASE				:				str				      = None
           if self.use_token_type_ids:
               SCREAMING_SNAKE_CASE				:				List[str]				      = ids_tensor([self.batch_size, self.seq_length]				,       self.type_vocab_size   )

           SCREAMING_SNAKE_CASE				:				Optional[int]				      = None
           SCREAMING_SNAKE_CASE				:				Any				      = None
           SCREAMING_SNAKE_CASE				:				List[str]				      = None
           if self.use_labels:
               SCREAMING_SNAKE_CASE				:				str				      = ids_tensor([self.batch_size]				,       self.type_sequence_label_size   )
               SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = ids_tensor([self.batch_size, self.seq_length]				,       self.num_labels   )
               SCREAMING_SNAKE_CASE				:				List[Any]				      = ids_tensor([self.batch_size]				,       self.num_choices   )

           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.get_config()

           return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






       def 	__lowerCAmelCase     (		self   )					->Dict:
           return LayoutLMConfig(
               vocab_size=self.vocab_size				,       hidden_size=self.hidden_size				,       num_hidden_layers=self.num_hidden_layers				,       num_attention_heads=self.num_attention_heads				,       intermediate_size=self.intermediate_size				,       hidden_act=self.hidden_act				,       hidden_dropout_prob=self.hidden_dropout_prob				,       attention_probs_dropout_prob=self.attention_probs_dropout_prob				,       max_position_embeddings=self.max_position_embeddings				,       type_vocab_size=self.type_vocab_size				,       initializer_range=self.initializer_range				,       )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase   )					->Optional[int]:
           SCREAMING_SNAKE_CASE				:				str				      = LayoutLMModel(config=lowercase_   )
           model.to(lowercase_   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				List[Any]				      = model(lowercase_				,       lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = model(lowercase_				,       lowercase_				,       token_type_ids=lowercase_   )
           SCREAMING_SNAKE_CASE				:				Any				      = model(lowercase_				,       lowercase_   )
           self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.hidden_size)   )
           self.parent.assertEqual(result.pooler_output.shape				,       (self.batch_size, self.hidden_size)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase   )					->Dict:
           SCREAMING_SNAKE_CASE				:				int				      = LayoutLMForMaskedLM(config=lowercase_   )
           model.to(lowercase_   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Dict				      = model(lowercase_				,       lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       labels=lowercase_   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.vocab_size)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase   )					->str:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.num_labels
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = LayoutLMForSequenceClassification(lowercase_   )
           model.to(lowercase_   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = model(
               lowercase_				,       lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       labels=lowercase_   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_labels)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase   )					->Union[str, Any]:
           SCREAMING_SNAKE_CASE				:				str				      = self.num_labels
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = LayoutLMForTokenClassification(config=lowercase_   )
           model.to(lowercase_   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				int				      = model(lowercase_				,       lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       labels=lowercase_   )
           self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.num_labels)   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase				,       _lowerCamelCase   )					->Union[str, Any]:
           SCREAMING_SNAKE_CASE				:				Any				      = LayoutLMForQuestionAnswering(config=lowercase_   )
           model.to(lowercase_   )
           model.eval()
           SCREAMING_SNAKE_CASE				:				List[Any]				      = model(
               lowercase_				,       bbox=lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       start_positions=lowercase_				,       end_positions=lowercase_				,       )
           self.parent.assertEqual(result.start_logits.shape				,       (self.batch_size, self.seq_length)   )
           self.parent.assertEqual(result.end_logits.shape				,       (self.batch_size, self.seq_length)   )






       def 	__lowerCAmelCase     (		self   )					->Optional[int]:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.prepare_config_and_inputs()
           (
               SCREAMING_SNAKE_CASE      
           )				:				Optional[int]				      = config_and_inputs
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = {
               '''input_ids''': input_ids,
               '''bbox''': bbox,
               '''token_type_ids''': token_type_ids,
               '''attention_mask''': input_mask,
           }
           return config, inputs_dict






@require_torch
class     a_       (			snake_case__				,    snake_case__				,    unittest.TestCase							):



       """simple docstring"""
       __SCREAMING_SNAKE_CASE       :					List[Any]						=  (
           (
               LayoutLMModel,
               LayoutLMForMaskedLM,
               LayoutLMForSequenceClassification,
               LayoutLMForTokenClassification,
               LayoutLMForQuestionAnswering,
           )
           if is_torch_available()
           else None
       )
       __SCREAMING_SNAKE_CASE       :					Dict						=  (
           {
               'document-question-answering': LayoutLMForQuestionAnswering,
               'feature-extraction': LayoutLMModel,
               'fill-mask': LayoutLMForMaskedLM,
               'text-classification': LayoutLMForSequenceClassification,
               'token-classification': LayoutLMForTokenClassification,
               'zero-shot': LayoutLMForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       __SCREAMING_SNAKE_CASE       :					List[Any]						=  True






       def 	__lowerCAmelCase     (		self   )					->str:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = LayoutLMModelTester(self   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = ConfigTester(self				,       config_class=lowercase_				,       hidden_size=37   )






       def 	__lowerCAmelCase     (		self   )					->Optional[int]:
           self.config_tester.run_common_tests()






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowercase_   )






       def 	__lowerCAmelCase     (		self   )					->List[Any]:
           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
               SCREAMING_SNAKE_CASE				:				List[Any]				      = type
               self.model_tester.create_and_check_model(*lowercase_   )






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           SCREAMING_SNAKE_CASE				:				Dict				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*lowercase_   )






       def 	__lowerCAmelCase     (		self   )					->str:
           SCREAMING_SNAKE_CASE				:				str				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*lowercase_   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*lowercase_   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*lowercase_   )






       @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''   )
       def 	__lowerCAmelCase     (		self   )					->List[str]:
           pass







def 				UpperCAmelCase_(    ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				List[Any]				      = torch.tensor([[101,1_019,1_014,1_016,1_037,12_849,4_747,1_004,14_246,2_278,5_439,4_524,5_002,2_930,2_193,2_930,4_341,3_208,1_005,1_055,2_171,2_848,11_300,3_531,102],[101,4_070,4_034,7_020,1_024,3_058,1_015,1_013,2_861,1_013,6_070,19_274,2_772,6_205,27_814,16_147,16_147,4_343,2_047,10_283,10_969,14_389,1_012,2_338,102]] ,						device=a__   )  # noqa: E231
    SCREAMING_SNAKE_CASE				:				List[str]				      = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],] ,						device=a__   )  # noqa: E231
    SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1_000,1_000,1_000,1_000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1_000,1_000,1_000,1_000]]] ,						device=a__   )  # noqa: E231
    SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]] ,						device=a__   )  # noqa: E231
    # these are sequence labels (i.e. at the token level)
    SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]] ,						device=a__   )  # noqa: E231
    # fmt: on

    return input_ids, attention_mask, bbox, token_type_ids, labels






@require_torch
class     a_       (			unittest.TestCase							):



       """simple docstring"""






       @slow
       def 	__lowerCAmelCase     (		self   )					->Optional[Any]:
           SCREAMING_SNAKE_CASE				:				str				      = LayoutLMModel.from_pretrained('''microsoft/layoutlm-base-uncased'''   ).to(lowercase_   )

           SCREAMING_SNAKE_CASE				:				Tuple				      = prepare_layoutlm_batch_inputs()

           # forward pass
           SCREAMING_SNAKE_CASE				:				Dict				      = model(input_ids=lowercase_				,       bbox=lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_   )

           # test the sequence output on [0, :3, :3]
           SCREAMING_SNAKE_CASE				:				str				      = torch.tensor(
               [[0.1_7_8_5, -0.1_9_4_7, -0.0_4_2_5], [-0.3_2_5_4, -0.2_8_0_7, 0.2_5_5_3], [-0.5_3_9_1, -0.3_3_2_2, 0.3_3_6_4]]				,       device=lowercase_				,       )

           self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]				,       lowercase_				,       atol=1e-3   )   )

           # test the pooled output on [1, :3]
           SCREAMING_SNAKE_CASE				:				str				      = torch.tensor([-0.6_5_8_0, -0.0_2_1_4, 0.8_5_5_2]				,       device=lowercase_   )

           self.assertTrue(torch.allclose(outputs.pooler_output[1, :3]				,       lowercase_				,       atol=1e-3   )   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->Optional[Any]:
           # initialize model with randomly initialized sequence classification head
           SCREAMING_SNAKE_CASE				:				int				      = LayoutLMForSequenceClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''				,       num_labels=2   ).to(
               lowercase_   )

           SCREAMING_SNAKE_CASE				:				Any				      = prepare_layoutlm_batch_inputs()

           # forward pass
           SCREAMING_SNAKE_CASE				:				str				      = model(
               input_ids=lowercase_				,       bbox=lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       labels=torch.tensor([1, 1]				,       device=lowercase_   )				,       )

           # test whether we get a loss as a scalar
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = outputs.loss
           SCREAMING_SNAKE_CASE				:				Tuple				      = torch.Size([]   )
           self.assertEqual(loss.shape				,       lowercase_   )

           # test the shape of the logits
           SCREAMING_SNAKE_CASE				:				Any				      = outputs.logits
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = torch.Size((2, 2)   )
           self.assertEqual(logits.shape				,       lowercase_   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->List[str]:
           # initialize model with randomly initialized token classification head
           SCREAMING_SNAKE_CASE				:				List[str]				      = LayoutLMForTokenClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''				,       num_labels=13   ).to(
               lowercase_   )

           SCREAMING_SNAKE_CASE				:				Any				      = prepare_layoutlm_batch_inputs()

           # forward pass
           SCREAMING_SNAKE_CASE				:				List[Any]				      = model(
               input_ids=lowercase_				,       bbox=lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_				,       labels=lowercase_   )

           # test the loss calculation to be around 2.65
           # expected_loss = torch.tensor(2.65, device=torch_device)

           # The loss is currently somewhat random and can vary between 0.1-0.3 atol.
           # self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

           # test the shape of the logits
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = outputs.logits
           SCREAMING_SNAKE_CASE				:				List[str]				      = torch.Size((2, 25, 13)   )
           self.assertEqual(logits.shape				,       lowercase_   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->str:
           # initialize model with randomly initialized token classification head
           SCREAMING_SNAKE_CASE				:				Tuple				      = LayoutLMForQuestionAnswering.from_pretrained('''microsoft/layoutlm-base-uncased'''   ).to(lowercase_   )

           SCREAMING_SNAKE_CASE				:				Any				      = prepare_layoutlm_batch_inputs()

           # forward pass
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = model(input_ids=lowercase_				,       bbox=lowercase_				,       attention_mask=lowercase_				,       token_type_ids=lowercase_   )

           # test the shape of the logits
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = torch.Size((2, 25)   )
           self.assertEqual(outputs.start_logits.shape				,       lowercase_   )
           self.assertEqual(outputs.end_logits.shape				,       lowercase_   )



"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


_SCREAMING_SNAKE_CASE							:	Union[str, Any]										=			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE							:	Union[str, Any]										=			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE							:	List[Any]										=			{
    """vocab_file""": {
        """microsoft/mpnet-base""": """https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """microsoft/mpnet-base""": """https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json""",
    },
}

_SCREAMING_SNAKE_CASE							:	List[Any]										=			{
    """microsoft/mpnet-base""": 5_1_2,
}

_SCREAMING_SNAKE_CASE							:	Optional[int]										=			{
    """microsoft/mpnet-base""": {"""do_lower_case""": True},
}

class      __a (   snake_case__     ):

			"""simple docstring"""

			SCREAMING_SNAKE_CASE_      =    VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE_      =    PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE_      =    PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE_      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE_      =    MPNetTokenizer
			SCREAMING_SNAKE_CASE_      =    ['input_ids', 'attention_mask']






			def __init__(				self  :       List[str]    ,	lowercase_  :       List[Any]=None    ,	lowercase_  :       Optional[int]=None    ,	lowercase_  :       Optional[int]=True    ,	lowercase_  :       str="<s>"    ,	lowercase_  :       str="</s>"    ,	lowercase_  :       List[Any]="</s>"    ,	lowercase_  :       Tuple="<s>"    ,	lowercase_  :       Any="[UNK]"    ,	lowercase_  :       Dict="<pad>"    ,	lowercase_  :       Optional[Any]="<mask>"    ,	lowercase_  :       List[str]=True    ,	lowercase_  :       Optional[Any]=None    ,	**lowercase_  :       Dict    ,	):
								super().__init__(
								    lowercase_    ,	tokenizer_file=lowercase_    ,	do_lower_case=lowercase_    ,	bos_token=lowercase_    ,	eos_token=lowercase_    ,	sep_token=lowercase_    ,	cls_token=lowercase_    ,	unk_token=lowercase_    ,	pad_token=lowercase_    ,	mask_token=lowercase_    ,	tokenize_chinese_chars=lowercase_    ,	strip_accents=lowercase_    ,	**lowercase_    ,	)

								UpperCamelCase__		:     List[str]								=json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
								if (
								    pre_tok_state.get('''lowercase'''    ,	lowercase_    ) != do_lower_case
								    or pre_tok_state.get('''strip_accents'''    ,	lowercase_    ) != strip_accents
								):
													UpperCamelCase__		:     int								=getattr(lowercase_    ,	pre_tok_state.pop('''type'''    )    )
													UpperCamelCase__		:     List[Any]								=do_lower_case
													UpperCamelCase__		:     Optional[Any]								=strip_accents
													UpperCamelCase__		:     Optional[int]								=pre_tok_class(**lowercase_    )

								UpperCamelCase__		:     Tuple								=do_lower_case






			@property
			def  _lowerCAmelCase      (				self  :       List[str]    ):
								if self._mask_token is None:
													if self.verbose:
																		logger.error('''Using mask_token, but it is not set yet.'''    )
													return None
								return str(self._mask_token    )






			@mask_token.setter
			def  _lowerCAmelCase      (				self  :       int    ,	lowercase_  :       str    ):
								UpperCamelCase__		:     int								=AddedToken(lowercase_    ,	lstrip=lowercase_    ,	rstrip=lowercase_    ) if isinstance(lowercase_    ,	lowercase_    ) else value
								UpperCamelCase__		:     str								=value






			def  _lowerCAmelCase      (				self  :       Tuple    ,	lowercase_  :       Any    ,	lowercase_  :       Tuple=None    ):
								UpperCamelCase__		:     int								=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
													return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






			def  _lowerCAmelCase      (				self  :       List[str]    ,	lowercase_  :       List[int]    ,	lowercase_  :       Optional[List[int]] = None    ):
								UpperCamelCase__		:     str								=[self.sep_token_id]
								UpperCamelCase__		:     List[str]								=[self.cls_token_id]

								if token_ids_a is None:
													return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






			def  _lowerCAmelCase      (				self  :       str    ,	lowercase_  :       str    ,	lowercase_  :       Optional[str] = None    ):
								UpperCamelCase__		:     List[Any]								=self._tokenizer.model.save(lowercase_    ,	name=lowercase_    )
								return tuple(lowercase_    )


