




from __future__ import annotations

import unittest

from transformers import DebertaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TFDebertaForMaskedLM,
	    TFDebertaForQuestionAnswering,
	    TFDebertaForSequenceClassification,
	    TFDebertaForTokenClassification,
	    TFDebertaModel,
	)

class 		lowercase_       :




					def __init__(   self:		Tuple,     _lowercase:		Tuple,     _lowercase:		Union[str, Any]=13,     _lowercase:		str=7,     _lowercase:		Dict=True,     _lowercase:		List[str]=True,     _lowercase:		Tuple=True,     _lowercase:		Optional[Any]=True,     _lowercase:		Optional[int]=99,     _lowercase:		Dict=32,     _lowercase:		Any=2,     _lowercase:		Optional[int]=4,     _lowercase:		List[Any]=37,     _lowercase:		Optional[Any]="gelu",     _lowercase:		Dict=0.1,     _lowercase:		Tuple=0.1,     _lowercase:		Any=512,     _lowercase:		List[Any]=16,     _lowercase:		List[str]=2,     _lowercase:		Any=0.02,     _lowercase:		Any=3,     _lowercase:		List[str]=4,     _lowercase:		List[str]=None,     ):


									'''simple docstring'''


									__lowerCAmelCase					=  parent
									__lowerCAmelCase					=  13
									__lowerCAmelCase					=  7
									__lowerCAmelCase					=  True
									__lowerCAmelCase					=  True
									__lowerCAmelCase					=  True
									__lowerCAmelCase					=  True
									__lowerCAmelCase					=  99
									__lowerCAmelCase					=  32
									__lowerCAmelCase					=  2
									__lowerCAmelCase					=  4
									__lowerCAmelCase					=  37
									__lowerCAmelCase					=  """gelu"""
									__lowerCAmelCase					=  0.1
									__lowerCAmelCase					=  0.1
									__lowerCAmelCase					=  512
									__lowerCAmelCase					=  16
									__lowerCAmelCase					=  False
									__lowerCAmelCase					=  -1
									__lowerCAmelCase					=  True
									__lowerCAmelCase					=  2
									__lowerCAmelCase					=  0.02
									__lowerCAmelCase					=  3
									__lowerCAmelCase					=  4
									__lowerCAmelCase					=  None




					def     _lowercase						(   self:		Any):


									'''simple docstring'''


									__lowerCAmelCase					=  ids_tensor([self.batch_size, self.seq_length],     self.vocab_size)

									__lowerCAmelCase					=  None
									if self.use_input_mask:
													__lowerCAmelCase					=  random_attention_mask([self.batch_size, self.seq_length])

									__lowerCAmelCase					=  None
									if self.use_token_type_ids:
													__lowerCAmelCase					=  ids_tensor([self.batch_size, self.seq_length],     self.type_vocab_size)

									__lowerCAmelCase					=  None
									__lowerCAmelCase					=  None
									__lowerCAmelCase					=  None
									if self.use_labels:
													__lowerCAmelCase					=  ids_tensor([self.batch_size],     self.type_sequence_label_size)
													__lowerCAmelCase					=  ids_tensor([self.batch_size, self.seq_length],     self.num_labels)

									__lowerCAmelCase					=  DebertaConfig(
									    vocab_size=self.vocab_size,     hidden_size=self.hidden_size,     num_hidden_layers=self.num_hidden_layers,     num_attention_heads=self.num_attention_heads,     intermediate_size=self.intermediate_size,     hidden_act=self.hidden_act,     hidden_dropout_prob=self.hidden_dropout_prob,     attention_probs_dropout_prob=self.attention_probs_dropout_prob,     max_position_embeddings=self.max_position_embeddings,     type_vocab_size=self.type_vocab_size,     relative_attention=self.relative_attention,     max_relative_positions=self.max_relative_positions,     position_biased_input=self.position_biased_input,     initializer_range=self.initializer_range,     return_dict=_lowercase,     )

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




					def     _lowercase						(   self:		List[str],     _lowercase:		Any,     _lowercase:		List[Any],     _lowercase:		Dict,     _lowercase:		List[Any],     _lowercase:		Optional[Any],     _lowercase:		List[str],     _lowercase:		Any):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaModel(config=_lowercase)
									__lowerCAmelCase					=  {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

									__lowerCAmelCase					=  [input_ids, input_mask]
									__lowerCAmelCase					=  model(_lowercase)

									__lowerCAmelCase					=  model(_lowercase)

									self.parent.assertEqual(result.last_hidden_state.shape,     (self.batch_size, self.seq_length, self.hidden_size))




					def     _lowercase						(   self:		List[str],     _lowercase:		Any,     _lowercase:		Dict,     _lowercase:		Union[str, Any],     _lowercase:		List[str],     _lowercase:		int,     _lowercase:		Optional[int],     _lowercase:		Optional[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaForMaskedLM(config=_lowercase)
									__lowerCAmelCase					=  {
									    """input_ids""": input_ids,
									    """attention_mask""": input_mask,
									    """token_type_ids""": token_type_ids,
									}
									__lowerCAmelCase					=  model(_lowercase)
									self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.seq_length, self.vocab_size))




					def     _lowercase						(   self:		Optional[Any],     _lowercase:		Optional[int],     _lowercase:		Optional[Any],     _lowercase:		Union[str, Any],     _lowercase:		Tuple,     _lowercase:		List[Any],     _lowercase:		Dict,     _lowercase:		Any):


									'''simple docstring'''


									__lowerCAmelCase					=  self.num_labels
									__lowerCAmelCase					=  TFDebertaForSequenceClassification(config=_lowercase)
									__lowerCAmelCase					=  {
									    """input_ids""": input_ids,
									    """attention_mask""": input_mask,
									    """token_type_ids""": token_type_ids,
									}

									__lowerCAmelCase					=  model(_lowercase)
									self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.num_labels))




					def     _lowercase						(   self:		Tuple,     _lowercase:		int,     _lowercase:		Optional[Any],     _lowercase:		int,     _lowercase:		List[str],     _lowercase:		Dict,     _lowercase:		Dict,     _lowercase:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.num_labels
									__lowerCAmelCase					=  TFDebertaForTokenClassification(config=_lowercase)
									__lowerCAmelCase					=  {
									    """input_ids""": input_ids,
									    """attention_mask""": input_mask,
									    """token_type_ids""": token_type_ids,
									}
									__lowerCAmelCase					=  model(_lowercase)
									self.parent.assertEqual(result.logits.shape,     (self.batch_size, self.seq_length, self.num_labels))




					def     _lowercase						(   self:		Tuple,     _lowercase:		Dict,     _lowercase:		Optional[Any],     _lowercase:		Tuple,     _lowercase:		Union[str, Any],     _lowercase:		Optional[int],     _lowercase:		List[str],     _lowercase:		Any):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaForQuestionAnswering(config=_lowercase)
									__lowerCAmelCase					=  {
									    """input_ids""": input_ids,
									    """attention_mask""": input_mask,
									    """token_type_ids""": token_type_ids,
									}

									__lowerCAmelCase					=  model(_lowercase)
									self.parent.assertEqual(result.start_logits.shape,     (self.batch_size, self.seq_length))
									self.parent.assertEqual(result.end_logits.shape,     (self.batch_size, self.seq_length))







					def     _lowercase						(   self:		Optional[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.prepare_config_and_inputs()
									(
									    (
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				(
									    __lowerCAmelCase
									)	,				
									)					=  config_and_inputs
									__lowerCAmelCase					=  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
									return config, inputs_dict





@require_tf
class 		lowercase_       (	lowerCAmelCase__						,							lowerCAmelCase__						,							unittest.TestCase				):
					__UpperCamelCase     	= (
					    (
					        TFDebertaModel,
					        TFDebertaForMaskedLM,
					        TFDebertaForQuestionAnswering,
					        TFDebertaForSequenceClassification,
					        TFDebertaForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					__UpperCamelCase     	= (
					    {
					        "feature-extraction": TFDebertaModel,
					        "fill-mask": TFDebertaForMaskedLM,
					        "question-answering": TFDebertaForQuestionAnswering,
					        "text-classification": TFDebertaForSequenceClassification,
					        "token-classification": TFDebertaForTokenClassification,
					        "zero-shot": TFDebertaForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)

					__UpperCamelCase     	= False
					__UpperCamelCase     	= False




					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaModelTester(self)
									__lowerCAmelCase					=  ConfigTester(self,     config_class=_lowercase,     hidden_size=37)




					def     _lowercase						(   self:		Union[str, Any]):


									'''simple docstring'''


									self.config_tester.run_common_tests()




					def     _lowercase						(   self:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_lowercase)




					def     _lowercase						(   self:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_lowercase)




					def     _lowercase						(   self:		List[str]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_lowercase)




					def     _lowercase						(   self:		Union[str, Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_lowercase)




					def     _lowercase						(   self:		str):


									'''simple docstring'''


									__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_lowercase)







					@slow
					def     _lowercase						(   self:		Any):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaModel.from_pretrained("""kamalkraj/deberta-base""")
									self.assertIsNotNone(_lowercase)





@require_tf
class 		lowercase_       (	unittest.TestCase				):




					@unittest.skip(reason="""Model not available yet""")
					def     _lowercase						(   self:		Optional[Any]):


									'''simple docstring'''


									pass







					@slow
					def     _lowercase						(   self:		List[str]):


									'''simple docstring'''


									__lowerCAmelCase					=  TFDebertaModel.from_pretrained("""kamalkraj/deberta-base""")
									__lowerCAmelCase					=  tf.constant([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
									__lowerCAmelCase					=  tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
									__lowerCAmelCase					=  model(_lowercase,     attention_mask=_lowercase)[0]

									__lowerCAmelCase					=  tf.constant(
									    [
									        [
									            [-0.59_855_896, -0.80_552_566, -0.8_462_135],
									            [1.4_484_025, -0.93_483_794, -0.80_593_085],
									            [0.3_122_741, 0.00_316_059, -1.4_131_377],
									        ]
									    ])
									tf.debugging.assert_near(output[:, 1:4, 1:4],     _lowercase,     atol=1e-4)













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
	UpperCAmelCase_					:		str           = None


UpperCAmelCase_					:		int           = logging.get_logger(__name__)

UpperCAmelCase_					:		List[str]           = {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase_					:		Any           = {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase_					:		Any           = {
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}

class        _SCREAMING_SNAKE_CASE			(     _a ):
	snake_case__				:	Dict	      =    VOCAB_FILES_NAMES
	snake_case__				:	Optional[int]	      =    PRETRAINED_VOCAB_FILES_MAP
	snake_case__				:	str	      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__				:	int	      =    ["""input_ids""", """attention_mask"""]
	snake_case__				:	Dict	      =    XLMRobertaTokenizer





	def __init__(						self							:   List[str]					,    __lowerCamelCase							:   List[Any]=None					,    __lowerCamelCase							:   int=None					,    __lowerCamelCase							:   Dict="<s>"					,    __lowerCamelCase							:   Optional[int]="</s>"					,    __lowerCamelCase							:   int="</s>"					,    __lowerCamelCase							:   Dict="<s>"					,    __lowerCamelCase							:   Optional[Any]="<unk>"					,    __lowerCamelCase							:   Union[str, Any]="<pad>"					,    __lowerCamelCase							:   Any="<mask>"					,    **__lowerCamelCase							:   List[str]					,    ):
		# Mask token behave like a normal word, i.e. include the space before it
		UpperCamelCase	:int		=      AddedToken(__lowerCamelCase					,    lstrip=__lowerCamelCase					,    rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase					,    __lowerCamelCase      ) else mask_token

		super().__init__(
		    __lowerCamelCase					,    tokenizer_file=__lowerCamelCase					,    bos_token=__lowerCamelCase					,    eos_token=__lowerCamelCase					,    sep_token=__lowerCamelCase					,    cls_token=__lowerCamelCase					,    unk_token=__lowerCamelCase					,    pad_token=__lowerCamelCase					,    mask_token=__lowerCamelCase					,    **__lowerCamelCase					,    )

		UpperCamelCase	:Optional[int]		=      vocab_file
		UpperCamelCase	:List[Any]		=      False if not self.vocab_file else True





	def 		_A					(						self							:   Dict					,    __lowerCamelCase							:   List[int]					,    __lowerCamelCase							:   Optional[List[int]] = None      ):

		if token_ids_a is None:
			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
		UpperCamelCase	:int		=      [self.cls_token_id]
		UpperCamelCase	:Any		=      [self.sep_token_id]
		return cls + token_ids_a + sep + sep + token_ids_a + sep





	def 		_A					(						self							:   List[str]					,    __lowerCamelCase							:   List[int]					,    __lowerCamelCase							:   Optional[List[int]] = None      ):
		UpperCamelCase	:str		=      [self.sep_token_id]
		UpperCamelCase	:int		=      [self.cls_token_id]

		if token_ids_a is None:
			return len(cls + token_ids_a + sep      ) * [0]
		return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]





	def 		_A					(						self							:   Optional[int]					,    __lowerCamelCase							:   str					,    __lowerCamelCase							:   Optional[str] = None      ):
		if not self.can_save_slow_tokenizer:
			raise ValueError(
			    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
			    """tokenizer."""      )

		if not os.path.isdir(__lowerCamelCase      ):
			logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""      )
			return
		UpperCamelCase	:Union[str, Any]		=      os.path.join(
		    __lowerCamelCase					,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

		if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowerCamelCase      ):
			copyfile(self.vocab_file					,    __lowerCamelCase      )

		return (out_vocab_file,)
