




"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
      SCREAMING_SNAKE_CASE							=			None


SCREAMING_SNAKE_CASE							=			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							=			{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE							=			{
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE							=			{
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}




class       UpperCAmelCase_    (		A_				):
    lowercase__		   = VOCAB_FILES_NAMES
    lowercase__		   = PRETRAINED_VOCAB_FILES_MAP
    lowercase__		   = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__		   = ['''input_ids''', '''attention_mask''']
    lowercase__		   = XLMRobertaTokenizer




    def __init__(       self		:			Dict      ,					snake_case_		:			Tuple=None      ,					snake_case_		:			Tuple=None      ,					snake_case_		:			Dict="<s>"      ,					snake_case_		:			Dict="</s>"      ,					snake_case_		:			List[str]="</s>"      ,					snake_case_		:			List[str]="<s>"      ,					snake_case_		:			Union[str, Any]="<unk>"      ,					snake_case_		:			List[str]="<pad>"      ,					snake_case_		:			Tuple="<mask>"      ,					**snake_case_		:			Optional[int]      ,					) ->				List[Any]:




     '''simple docstring'''

     A__			=							AddedToken(snake_case_      ,					lstrip=snake_case_      ,					rstrip=snake_case_ ) if isinstance(snake_case_      ,					snake_case_ ) else mask_token

     super().__init__(
         snake_case_      ,					tokenizer_file=snake_case_      ,					bos_token=snake_case_      ,					eos_token=snake_case_      ,					sep_token=snake_case_      ,					cls_token=snake_case_      ,					unk_token=snake_case_      ,					pad_token=snake_case_      ,					mask_token=snake_case_      ,					**snake_case_      ,					)

     A__			=							vocab_file
     A__			=							False if not self.vocab_file else True




    def 				__magic_name__				(       self		:			str      ,					snake_case_		:			List[int]      ,					snake_case_		:			Optional[List[int]] = None ) ->				List[int]:




     '''simple docstring'''

     if token_ids_a is None:
      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     A__			=							[self.cls_token_id]
     A__			=							[self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep




    def 				__magic_name__				(       self		:			Optional[int]      ,					snake_case_		:			List[int]      ,					snake_case_		:			Optional[List[int]] = None ) ->				List[int]:




     '''simple docstring'''

     A__			=							[self.sep_token_id]
     A__			=							[self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep ) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]




    def 				__magic_name__				(       self		:			Dict      ,					snake_case_		:			str      ,					snake_case_		:			Optional[str] = None ) ->				Tuple[str]:




     '''simple docstring'''

     if not self.can_save_slow_tokenizer:
      raise ValueError(
          "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
          "tokenizer." )

     if not os.path.isdir(snake_case_ ):
      logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""" )
      return
     A__			=							os.path.join(
         snake_case_      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

     if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case_ ):
      copyfile(self.vocab_file      ,					snake_case_ )

     return (out_vocab_file,)











"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers.models.mpnet.modeling_tf_mpnet import (
          TFMPNetForMaskedLM,
          TFMPNetForMultipleChoice,
          TFMPNetForQuestionAnswering,
          TFMPNetForSequenceClassification,
          TFMPNetForTokenClassification,
          TFMPNetModel,
      )




class       UpperCAmelCase_    :




    def __init__(       self		:			Dict      ,					snake_case_		:			List[str]      ,					snake_case_		:			List[Any]=13      ,					snake_case_		:			int=7      ,					snake_case_		:			List[Any]=True      ,					snake_case_		:			Tuple=True      ,					snake_case_		:			Tuple=False      ,					snake_case_		:			str=True      ,					snake_case_		:			str=99      ,					snake_case_		:			int=64      ,					snake_case_		:			Tuple=2      ,					snake_case_		:			Optional[int]=4      ,					snake_case_		:			List[Any]=64      ,					snake_case_		:			Any="gelu"      ,					snake_case_		:			List[Any]=0.1      ,					snake_case_		:			List[Any]=0.1      ,					snake_case_		:			Dict=512      ,					snake_case_		:			Dict=16      ,					snake_case_		:			Tuple=2      ,					snake_case_		:			Tuple=0.02      ,					snake_case_		:			Optional[Any]=3      ,					snake_case_		:			Dict=4      ,					snake_case_		:			List[str]=None      ,					) ->				Any:




     '''simple docstring'''

     A__			=							parent
     A__			=							batch_size
     A__			=							seq_length
     A__			=							is_training
     A__			=							use_input_mask
     A__			=							use_token_type_ids
     A__			=							use_labels
     A__			=							vocab_size
     A__			=							hidden_size
     A__			=							num_hidden_layers
     A__			=							num_attention_heads
     A__			=							intermediate_size
     A__			=							hidden_act
     A__			=							hidden_dropout_prob
     A__			=							attention_probs_dropout_prob
     A__			=							max_position_embeddings
     A__			=							type_vocab_size
     A__			=							type_sequence_label_size
     A__			=							initializer_range
     A__			=							num_labels
     A__			=							num_choices
     A__			=							scope




    def 				__magic_name__				(       self		:			int ) ->				Optional[int]:




     '''simple docstring'''

     A__			=							ids_tensor([self.batch_size, self.seq_length]      ,					self.vocab_size )

     A__			=							None
     if self.use_input_mask:
      A__			=							random_attention_mask([self.batch_size, self.seq_length] )

     A__			=							None
     A__			=							None
     A__			=							None
     if self.use_labels:
      A__			=							ids_tensor([self.batch_size]      ,					self.type_sequence_label_size )
      A__			=							ids_tensor([self.batch_size, self.seq_length]      ,					self.num_labels )
      A__			=							ids_tensor([self.batch_size]      ,					self.num_choices )

     A__			=							MPNetConfig(
         vocab_size=self.vocab_size      ,					hidden_size=self.hidden_size      ,					num_hidden_layers=self.num_hidden_layers      ,					num_attention_heads=self.num_attention_heads      ,					intermediate_size=self.intermediate_size      ,					hidden_act=self.hidden_act      ,					hidden_dropout_prob=self.hidden_dropout_prob      ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,					max_position_embeddings=self.max_position_embeddings      ,					initializer_range=self.initializer_range      ,					)
     return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




    def 				__magic_name__				(       self		:			str      ,					snake_case_		:			int      ,					snake_case_		:			int      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			Optional[Any]      ,					snake_case_		:			Any      ,					snake_case_		:			str ) ->				int:




     '''simple docstring'''

     A__			=							TFMPNetModel(config=snake_case_ )
     A__			=							{"input_ids": input_ids, "attention_mask": input_mask}
     A__			=							model(snake_case_ )
     A__			=							[input_ids, input_mask]
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size) )




    def 				__magic_name__				(       self		:			List[Any]      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			List[str]      ,					snake_case_		:			Optional[int]      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			str      ,					snake_case_		:			str ) ->				int:




     '''simple docstring'''

     A__			=							TFMPNetForMaskedLM(config=snake_case_ )
     A__			=							{"input_ids": input_ids, "attention_mask": input_mask}
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size) )




    def 				__magic_name__				(       self		:			List[Any]      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			Tuple      ,					snake_case_		:			Any      ,					snake_case_		:			str      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			Dict ) ->				Tuple:




     '''simple docstring'''

     A__			=							TFMPNetForQuestionAnswering(config=snake_case_ )
     A__			=							{
         "input_ids": input_ids,
         "attention_mask": input_mask,
     }
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.start_logits.shape      ,					(self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape      ,					(self.batch_size, self.seq_length) )




    def 				__magic_name__				(       self		:			List[Any]      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			List[str]      ,					snake_case_		:			Dict      ,					snake_case_		:			Optional[int]      ,					snake_case_		:			Optional[int]      ,					snake_case_		:			Optional[int] ) ->				Tuple:




     '''simple docstring'''

     A__			=							self.num_labels
     A__			=							TFMPNetForSequenceClassification(snake_case_ )
     A__			=							{"input_ids": input_ids, "attention_mask": input_mask}
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_labels) )




    def 				__magic_name__				(       self		:			Dict      ,					snake_case_		:			Tuple      ,					snake_case_		:			int      ,					snake_case_		:			List[Any]      ,					snake_case_		:			Optional[int]      ,					snake_case_		:			str      ,					snake_case_		:			Dict ) ->				Optional[Any]:




     '''simple docstring'''

     A__			=							self.num_choices
     A__			=							TFMPNetForMultipleChoice(snake_case_ )
     A__			=							tf.tile(tf.expand_dims(snake_case_      ,					1 )      ,					(1, self.num_choices, 1) )
     A__			=							tf.tile(tf.expand_dims(snake_case_      ,					1 )      ,					(1, self.num_choices, 1) )
     A__			=							{
         "input_ids": multiple_choice_inputs_ids,
         "attention_mask": multiple_choice_input_mask,
     }
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.num_choices) )




    def 				__magic_name__				(       self		:			List[str]      ,					snake_case_		:			str      ,					snake_case_		:			Tuple      ,					snake_case_		:			Union[str, Any]      ,					snake_case_		:			List[str]      ,					snake_case_		:			Optional[Any]      ,					snake_case_		:			List[Any] ) ->				int:




     '''simple docstring'''

     A__			=							self.num_labels
     A__			=							TFMPNetForTokenClassification(snake_case_ )
     A__			=							{"input_ids": input_ids, "attention_mask": input_mask}
     A__			=							model(snake_case_ )
     self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.num_labels) )




    def 				__magic_name__				(       self		:			Tuple ) ->				Dict:




     '''simple docstring'''

     A__			=							self.prepare_config_and_inputs()
     ((A__), (A__), (A__), (A__), (A__), (A__))			=							config_and_inputs
     A__			=							{"input_ids": input_ids, "attention_mask": input_mask}
     return config, inputs_dict






@require_tf
class       UpperCAmelCase_    (		A_,  A_,  unittest.TestCase				):
    lowercase__		   = (
        (
            TFMPNetForMaskedLM,
            TFMPNetForMultipleChoice,
            TFMPNetForQuestionAnswering,
            TFMPNetForSequenceClassification,
            TFMPNetForTokenClassification,
            TFMPNetModel,
        )
        if is_tf_available()
        else ()
    )
    lowercase__		   = (
        {
            '''feature-extraction''': TFMPNetModel,
            '''fill-mask''': TFMPNetForMaskedLM,
            '''question-answering''': TFMPNetForQuestionAnswering,
            '''text-classification''': TFMPNetForSequenceClassification,
            '''token-classification''': TFMPNetForTokenClassification,
            '''zero-shot''': TFMPNetForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    lowercase__		   = False
    lowercase__		   = False




    def 				__magic_name__				(       self		:			Any ) ->				List[str]:




     '''simple docstring'''

     A__			=							TFMPNetModelTester(self )
     A__			=							ConfigTester(self      ,					config_class=snake_case_      ,					hidden_size=37 )




    def 				__magic_name__				(       self		:			Union[str, Any] ) ->				Tuple:




     '''simple docstring'''

     self.config_tester.run_common_tests()




    def 				__magic_name__				(       self		:			Union[str, Any] ) ->				Dict:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_model(*snake_case_ )




    def 				__magic_name__				(       self		:			Optional[int] ) ->				List[Any]:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_for_masked_lm(*snake_case_ )




    def 				__magic_name__				(       self		:			Dict ) ->				Optional[Any]:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_for_question_answering(*snake_case_ )




    def 				__magic_name__				(       self		:			int ) ->				List[Any]:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_for_sequence_classification(*snake_case_ )




    def 				__magic_name__				(       self		:			Optional[Any] ) ->				int:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_for_multiple_choice(*snake_case_ )




    def 				__magic_name__				(       self		:			Tuple ) ->				Any:




     '''simple docstring'''

     A__			=							self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mpnet_for_token_classification(*snake_case_ )




    @slow
    def 				__magic_name__				(       self		:			Dict ) ->				Tuple:




     '''simple docstring'''

     for model_name in ["microsoft/mpnet-base"]:
      A__			=							TFMPNetModel.from_pretrained(snake_case_ )
      self.assertIsNotNone(snake_case_ )






@require_tf
class       UpperCAmelCase_    (		unittest.TestCase				):




    @slow
    def 				__magic_name__				(       self		:			str ) ->				List[str]:




     '''simple docstring'''

     A__			=							TFMPNetModel.from_pretrained("microsoft/mpnet-base" )
     A__			=							tf.constant([[0, 1, 2, 3, 4, 5]] )
     A__			=							model(snake_case_ )[0]

     A__			=							[1, 6, 768]
     self.assertEqual(output.shape      ,					snake_case_ )

     A__			=							tf.constant(
         [
             [
                 [-0.106_7172, 0.0821_6473, 0.002_4543],
                 [-0.0346_5879, 0.835_4118, -0.0325_2288],
                 [-0.0656_9476, -0.1242_4111, -0.049_4436],
             ]
         ] )
     tf.debugging.assert_near(output[:, :3, :3]      ,					snake_case_      ,					atol=1e-4 )





