


'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase:		int    						=  logging.get_logger(__name__)

__lowercase:		str    						=  {
    "google/pegasus-x-base": "https://huggingface.co/google/pegasus-x-base/resolve/main/config.json",
    "google/pegasus-x-large": "https://huggingface.co/google/pegasus-x-large/resolve/main/config.json",
    # See all PEGASUS-X models at https://huggingface.co/models?filter=pegasus-x
}







class 			UpperCAmelCase			(							SCREAMING_SNAKE_CASE__):
 _lowerCamelCase			:       Dict            =       'pegasus_x'
 _lowerCamelCase			:       Union[str, Any]            =       ['past_key_values']
 _lowerCamelCase			:       Any            =       {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}

 def __init__(			self			:						List[str],       a_			:						Any=9_6103,       a_			:						Union[str, Any]=1_6384,       a_			:						Optional[Any]=16,       a_			:						Tuple=4096,       a_			:						Optional[Any]=16,       a_			:						Optional[Any]=16,       a_			:						Tuple=4096,       a_			:						int=16,       a_			:						List[str]=0.0,       a_			:						Tuple=0.0,       a_			:						Tuple=True,       a_			:						str=True,       a_			:						int="gelu",       a_			:						Optional[Any]=1024,       a_			:						List[Any]=0.1,       a_			:						List[Any]=0.0,       a_			:						Any=0.0,       a_			:						str=0.02,       a_			:						Tuple=0,       a_			:						List[Any]=True,       a_			:						Tuple=0,       a_			:						Optional[int]=1,       a_			:						List[str]=1,       a_			:						Dict=32,       a_			:						Union[str, Any]=512,       a_			:						int=True,       **a_			:						List[Any],       ):





    """simple docstring"""



    UpperCamelCase__			    =    vocab_size
    UpperCamelCase__			    =    max_position_embeddings
    UpperCamelCase__			    =    d_model
    UpperCamelCase__			    =    encoder_ffn_dim
    UpperCamelCase__			    =    encoder_layers
    UpperCamelCase__			    =    encoder_attention_heads
    UpperCamelCase__			    =    decoder_ffn_dim
    UpperCamelCase__			    =    decoder_layers
    UpperCamelCase__			    =    decoder_attention_heads
    UpperCamelCase__			    =    dropout
    UpperCamelCase__			    =    attention_dropout
    UpperCamelCase__			    =    activation_dropout
    UpperCamelCase__			    =    activation_function
    UpperCamelCase__			    =    init_std
    UpperCamelCase__			    =    encoder_layerdrop
    UpperCamelCase__			    =    decoder_layerdrop
    UpperCamelCase__			    =    use_cache
    UpperCamelCase__			    =    encoder_layers
    UpperCamelCase__			    =    scale_embedding  # scale factor will be sqrt(d_model) if True

    UpperCamelCase__			    =    num_global_tokens
    UpperCamelCase__			    =    block_size
    UpperCamelCase__			    =    stagger_local_blocks

    super().__init__(
        pad_token_id=a_,       eos_token_id=a_,       is_encoder_decoder=a_,       decoder_start_token_id=a_,       forced_eos_token_id=a_,       **a_,       )

 @property
 def 		lowercase_    (			self			:						Union[str, Any]       ):





    """simple docstring"""



    return self.encoder_attention_heads

 @property
 def 		lowercase_    (			self			:						Optional[int]       ):





    """simple docstring"""



    return self.d_model






'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
		A :			Optional[Any]							   =      None


A :			Tuple							   =      logging.get_logger(__name__)

A :			Optional[int]							   =      {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

A :			Tuple							   =      {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

A :			int							   =      {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}







class       lowerCamelCase       (    __UpperCAmelCase	):
					_SCREAMING_SNAKE_CASE				 =     VOCAB_FILES_NAMES
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_VOCAB_FILES_MAP
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_SCREAMING_SNAKE_CASE				 =     ["input_ids", "attention_mask"]
					_SCREAMING_SNAKE_CASE				 =     XLMRobertaTokenizer


					def __init__(       self						:	Any							,  __snake_case						:	Tuple=None							,  __snake_case						:	List[Any]=None							,  __snake_case						:	List[str]="<s>"							,  __snake_case						:	int="</s>"							,  __snake_case						:	str="</s>"							,  __snake_case						:	Any="<s>"							,  __snake_case						:	int="<unk>"							,  __snake_case						:	Optional[Any]="<pad>"							,  __snake_case						:	Optional[int]="<mask>"							,  **__snake_case						:	List[str]							,  ):





											'''simple docstring'''





											_snake_case: Optional[Any]	       =		AddedToken(__snake_case							,  lstrip=__snake_case							,  rstrip=__snake_case							) if isinstance(__snake_case							,  __snake_case							) else mask_token

											super().__init__(
											    __snake_case							,  tokenizer_file=__snake_case							,  bos_token=__snake_case							,  eos_token=__snake_case							,  sep_token=__snake_case							,  cls_token=__snake_case							,  unk_token=__snake_case							,  pad_token=__snake_case							,  mask_token=__snake_case							,  **__snake_case							,  )

											_snake_case: int	       =		vocab_file
											_snake_case: List[Any]	       =		False if not self.vocab_file else True


					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							,  __snake_case						:	List[int]							,  __snake_case						:	Optional[List[int]] = None							):





											'''simple docstring'''





											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_snake_case: Union[str, Any]	       =		[self.cls_token_id]
											_snake_case: int	       =		[self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep


					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	List[int]							,  __snake_case						:	Optional[List[int]] = None							):





											'''simple docstring'''





											_snake_case: Union[str, Any]	       =		[self.sep_token_id]
											_snake_case: List[str]	       =		[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




					def  SCREAMING_SNAKE_CASE_		(       self						:	Any							,  __snake_case						:	str							,  __snake_case						:	Optional[str] = None							):





											'''simple docstring'''





											if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
																	    'tokenizer.'							)

											if not os.path.isdir(__snake_case							):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''							)
																	return
											_snake_case: Optional[int]	       =		os.path.join(
											    __snake_case							,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

											if os.path.abspath(self.vocab_file							) != os.path.abspath(__snake_case							):
																	copyfile(self.vocab_file							,  __snake_case							)

											return (out_vocab_file,)





