

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase							:  List[str]     =				logging.get_logger(__name__)
_lowerCamelCase							:  Any     =				{"vocab_file": "spiece.model"}

_lowerCamelCase							:  Tuple     =				{
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
}

_lowerCamelCase							:  List[str]     =				{
    "google/fnet-base": 5_1_2,
    "google/fnet-large": 5_1_2,
}

_lowerCamelCase							:  Union[str, Any]     =				"▁"





class    __UpperCAmelCase	(			lowerCamelCase__		):
   UpperCamelCase  	=	VOCAB_FILES_NAMES
   UpperCamelCase  	=	PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase  	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase  	=	["""input_ids""", """token_type_ids"""]



   def __init__(				self				:	Optional[Any],						__A				:	Tuple,						__A				:	List[str]=False,						__A				:	Union[str, Any]=True,						__A				:	List[Any]=True,						__A				:	Optional[Any]="<unk>",						__A				:	str="[SEP]",						__A				:	List[str]="<pad>",						__A				:	Optional[Any]="[CLS]",						__A				:	Dict="[MASK]",						__A				:	Optional[Dict[str, Any]] = None,						**__A				:	List[Any],						):
     # Mask token behave like a normal word, i.e. include the space before it and
     # is included in the raw text, there should be a match in a non-normalized sentence.
     UpperCAmelCase			:				str										=					(
         AddedToken(__A,						lstrip=__A,						rstrip=__A,						normalized=__A		)
         if isinstance(__A,						__A		)
         else mask_token
     )

     UpperCAmelCase			:				int										=					{} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=__A,						remove_space=__A,						keep_accents=__A,						unk_token=__A,						sep_token=__A,						pad_token=__A,						cls_token=__A,						mask_token=__A,						sp_model_kwargs=self.sp_model_kwargs,						**__A,						)

     UpperCAmelCase			:				Dict										=					do_lower_case
     UpperCAmelCase			:				Optional[Any]										=					remove_space
     UpperCAmelCase			:				str										=					keep_accents
     UpperCAmelCase			:				Optional[int]										=					vocab_file

     UpperCAmelCase			:				str										=					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
     self.sp_model.Load(__A		)



   @property
   def 			__magic_name__       (				self				:	str		):
     return len(self.sp_model		)



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				int										=					{self.convert_ids_to_tokens(__A		): i for i in range(self.vocab_size		)}
     vocab.update(self.added_tokens_encoder		)
     return vocab



   def __getstate__(				self				:	Dict		):
     UpperCAmelCase			:				List[str]										=					self.__dict__.copy()
     UpperCAmelCase			:				Union[str, Any]										=					None
     return state



   def __setstate__(				self				:	List[Any],						__A				:	int		):
     UpperCAmelCase			:				List[Any]										=					d

     # for backward compatibility
     if not hasattr(self,						'''sp_model_kwargs'''		):
       UpperCAmelCase			:				List[str]										=					{}

     UpperCAmelCase			:				int										=					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
     self.sp_model.Load(self.vocab_file		)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	List[Any]		):
     if self.remove_space:
       UpperCAmelCase			:				str										=					''' '''.join(inputs.strip().split()		)
     else:
       UpperCAmelCase			:				List[Any]										=					inputs
     UpperCAmelCase			:				List[str]										=					outputs.replace('''``''',						'''"'''		).replace('''\'\'''',						'''"'''		)

     if not self.keep_accents:
       UpperCAmelCase			:				List[Any]										=					unicodedata.normalize('''NFKD''',						__A		)
       UpperCAmelCase			:				Optional[int]										=					''''''.join([c for c in outputs if not unicodedata.combining(__A		)]		)
     if self.do_lower_case:
       UpperCAmelCase			:				List[Any]										=					outputs.lower()

     return outputs



   def 			__magic_name__       (				self				:	str,						__A				:	str		):
     UpperCAmelCase			:				Any										=					self.preprocess_text(__A		)
     UpperCAmelCase			:				str										=					self.sp_model.encode(__A,						out_type=__A		)
     UpperCAmelCase			:				str										=					[]
     for piece in pieces:
       if len(__A		) > 1 and piece[-1] == str(''','''		) and piece[-2].isdigit():
         UpperCAmelCase			:				Dict										=					self.sp_model.EncodeAsPieces(piece[:-1].replace(__A,						''''''		)		)
         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
           if len(cur_pieces[0]		) == 1:
             UpperCAmelCase			:				Dict										=					cur_pieces[1:]
           else:
             UpperCAmelCase			:				Optional[Any]										=					cur_pieces[0][1:]
         cur_pieces.append(piece[-1]		)
         new_pieces.extend(__A		)
       else:
         new_pieces.append(__A		)

     return new_pieces



   def 			__magic_name__       (				self				:	List[Any],						__A				:	Optional[int]		):
     return self.sp_model.PieceToId(__A		)



   def 			__magic_name__       (				self				:	List[str],						__A				:	List[str]		):
     return self.sp_model.IdToPiece(__A		)



   def 			__magic_name__       (				self				:	List[Any],						__A				:	Any		):
     UpperCAmelCase			:				Optional[Any]										=					[]
     UpperCAmelCase			:				Dict										=					''''''
     UpperCAmelCase			:				List[str]										=					False
     for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
         if not prev_is_special:
           out_string += " "
         out_string += self.sp_model.decode(__A		) + token
         UpperCAmelCase			:				Optional[Any]										=					True
         UpperCAmelCase			:				Dict										=					[]
       else:
         current_sub_tokens.append(__A		)
         UpperCAmelCase			:				Optional[int]										=					False
     out_string += self.sp_model.decode(__A		)
     return out_string.strip()



   def 			__magic_name__       (				self				:	str,						__A				:	List[int],						__A				:	bool = False,						__A				:	bool = None,						__A				:	bool = True,						**__A				:	Any,						):
     UpperCAmelCase			:				List[Any]										=					kwargs.pop('''use_source_tokenizer''',						__A		)

     UpperCAmelCase			:				List[Any]										=					self.convert_ids_to_tokens(__A,						skip_special_tokens=__A		)

     # To avoid mixing byte-level and unicode for byte-level BPT
     # we need to build string separately for added tokens and byte-level tokens
     # cf. https://github.com/huggingface/transformers/issues/1133
     UpperCAmelCase			:				Optional[int]										=					[]
     UpperCAmelCase			:				Union[str, Any]										=					[]
     for token in filtered_tokens:
       if skip_special_tokens and token in self.all_special_ids:
         continue
       if token in self.added_tokens_encoder:
         if current_sub_text:
           sub_texts.append(self.convert_tokens_to_string(__A		)		)
           UpperCAmelCase			:				Optional[Any]										=					[]
         sub_texts.append(__A		)
       else:
         current_sub_text.append(__A		)
     if current_sub_text:
       sub_texts.append(self.convert_tokens_to_string(__A		)		)

     # Mimic the behavior of the Rust tokenizer:
     # No space after <unk>
     if spaces_between_special_tokens:
       UpperCAmelCase			:				Union[str, Any]										=					re.sub(R'''(<unk>) ''',						R'''\1''',						''' '''.join(__A		)		)
     else:
       UpperCAmelCase			:				Optional[int]										=					''''''.join(__A		)

     UpperCAmelCase			:				int										=					(
         clean_up_tokenization_spaces
         if clean_up_tokenization_spaces is not None
         else self.clean_up_tokenization_spaces
     )
     if clean_up_tokenization_spaces:
       UpperCAmelCase			:				Optional[Any]										=					self.clean_up_tokenization(__A		)
       return clean_text
     else:
       return text



   def 			__magic_name__       (				self				:	Dict,						__A				:	List[int],						__A				:	Optional[List[int]] = None		):
     UpperCAmelCase			:				List[Any]										=					[self.sep_token_id]
     UpperCAmelCase			:				Tuple										=					[self.cls_token_id]
     if token_ids_a is None:
       return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep



   def 			__magic_name__       (				self				:	int,						__A				:	List[int],						__A				:	Optional[List[int]] = None,						__A				:	bool = False		):

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__A,						token_ids_a=__A,						already_has_special_tokens=__A		)

     if token_ids_a is not None:
       return [1] + ([0] * len(__A		)) + [1] + ([0] * len(__A		)) + [1]
     return [1] + ([0] * len(__A		)) + [1]



   def 			__magic_name__       (				self				:	Optional[Any],						__A				:	List[int],						__A				:	Optional[List[int]] = None		):
     UpperCAmelCase			:				List[Any]										=					[self.sep_token_id]
     UpperCAmelCase			:				Optional[Any]										=					[self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep		) * [0]
     return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





   def 			__magic_name__       (				self				:	Optional[Any],						__A				:	str,						__A				:	Optional[str] = None		):
     if not os.path.isdir(__A		):
       logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
       return
     UpperCAmelCase			:				List[str]										=					os.path.join(
         __A,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

     if os.path.abspath(self.vocab_file		) != os.path.abspath(__A		) and os.path.isfile(self.vocab_file		):
       copyfile(self.vocab_file,						__A		)
     elif not os.path.isfile(self.vocab_file		):
       with open(__A,						'''wb'''		) as fi:
         UpperCAmelCase			:				Union[str, Any]										=					self.sp_model.serialized_model_proto()
         fi.write(__A		)

     return (out_vocab_file,)








from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_lowerCamelCase							:  Union[str, Any]     =				logging.get_logger(__name__)

_lowerCamelCase							:  Optional[int]     =				{
    "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json",
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class    __UpperCAmelCase	(			lowerCamelCase__		):
   UpperCamelCase  	=	"""m2m_100"""
   UpperCamelCase  	=	["""past_key_values"""]
   UpperCamelCase  	=	{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



   def __init__(				self				:	int,						__A				:	Optional[Any]=1_2_8_1_1_2,						__A				:	str=1_0_2_4,						__A				:	Tuple=1_2,						__A				:	Any=4_0_9_6,						__A				:	str=1_6,						__A				:	List[str]=1_2,						__A				:	int=4_0_9_6,						__A				:	List[Any]=1_6,						__A				:	Tuple=0.0_5,						__A				:	Tuple=0.0_5,						__A				:	List[Any]=True,						__A				:	int=True,						__A				:	str="relu",						__A				:	str=1_0_2_4,						__A				:	str=0.1,						__A				:	str=0.1,						__A				:	List[str]=0.0,						__A				:	List[str]=0.0_2,						__A				:	Any=2,						__A				:	List[str]=True,						__A				:	Tuple=1,						__A				:	Optional[Any]=0,						__A				:	Optional[Any]=2,						**__A				:	Optional[Any],						):
     UpperCAmelCase			:				str										=					vocab_size
     UpperCAmelCase			:				Optional[Any]										=					max_position_embeddings
     UpperCAmelCase			:				List[str]										=					d_model
     UpperCAmelCase			:				Any										=					encoder_ffn_dim
     UpperCAmelCase			:				Optional[int]										=					encoder_layers
     UpperCAmelCase			:				List[Any]										=					encoder_attention_heads
     UpperCAmelCase			:				str										=					decoder_ffn_dim
     UpperCAmelCase			:				Any										=					decoder_layers
     UpperCAmelCase			:				List[str]										=					decoder_attention_heads
     UpperCAmelCase			:				str										=					dropout
     UpperCAmelCase			:				int										=					attention_dropout
     UpperCAmelCase			:				Dict										=					activation_dropout
     UpperCAmelCase			:				Tuple										=					activation_function
     UpperCAmelCase			:				Any										=					init_std
     UpperCAmelCase			:				Union[str, Any]										=					encoder_layerdrop
     UpperCAmelCase			:				List[Any]										=					decoder_layerdrop
     UpperCAmelCase			:				Optional[Any]										=					use_cache
     UpperCAmelCase			:				str										=					encoder_layers
     UpperCAmelCase			:				List[Any]										=					scale_embedding  # scale factor will be sqrt(d_model) if True

     super().__init__(
         pad_token_id=__A,						bos_token_id=__A,						eos_token_id=__A,						is_encoder_decoder=__A,						decoder_start_token_id=__A,						**__A,						)






class    __UpperCAmelCase	(			lowerCamelCase__		):



   @property
   def 			__magic_name__       (				self				:	List[str]		):
     UpperCAmelCase			:				List[Any]										=					OrderedDict(
         [
             ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
         ]		)

     if self.use_past:
       UpperCAmelCase			:				List[str]										=					{0: '''batch'''}
       UpperCAmelCase			:				Any										=					{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
     else:
       UpperCAmelCase			:				Any										=					{0: '''batch''', 1: '''decoder_sequence'''}
       UpperCAmelCase			:				List[Any]										=					{0: '''batch''', 1: '''decoder_sequence'''}

     if self.use_past:
       self.fill_with_past_key_values_(__A,						direction='''inputs'''		)
     return common_inputs



   def 			__magic_name__       (				self				:	Dict,						__A				:	PreTrainedTokenizer,						__A				:	int = -1,						__A				:	int = -1,						__A				:	bool = False,						__A				:	Optional[TensorType] = None,						):
     # Copied from OnnxConfig.generate_dummy_inputs
     # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
     # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
     UpperCAmelCase			:				List[str]										=					compute_effective_axis_dimension(
         __A,						fixed_dimension=OnnxConfig.default_fixed_batch,						num_token_to_add=0		)

     # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
     UpperCAmelCase			:				Optional[int]										=					tokenizer.num_special_tokens_to_add(__A		)
     UpperCAmelCase			:				int										=					compute_effective_axis_dimension(
         __A,						fixed_dimension=OnnxConfig.default_fixed_sequence,						num_token_to_add=__A		)

     # Generate dummy inputs according to compute batch and sequence
     UpperCAmelCase			:				List[str]										=					[''' '''.join([tokenizer.unk_token]		) * seq_length] * batch_size
     UpperCAmelCase			:				Tuple										=					dict(tokenizer(__A,						return_tensors=__A		)		)
     return common_inputs



   def 			__magic_name__       (				self				:	Any,						__A				:	PreTrainedTokenizer,						__A				:	int = -1,						__A				:	int = -1,						__A				:	bool = False,						__A				:	Optional[TensorType] = None,						):
     UpperCAmelCase			:				int										=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         __A,						__A,						__A,						__A,						__A		)

     # Generate decoder inputs
     UpperCAmelCase			:				Union[str, Any]										=					seq_length if not self.use_past else 1
     UpperCAmelCase			:				Optional[int]										=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         __A,						__A,						__A,						__A,						__A		)
     UpperCAmelCase			:				Dict										=					{F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
     UpperCAmelCase			:				Optional[int]										=					dict(**__A,						**__A		)

     if self.use_past:
       if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''		)
       else:
         import torch
       UpperCAmelCase					,		UpperCAmelCase			:				List[str]										=					common_inputs['''input_ids'''].shape
       UpperCAmelCase			:				Dict										=					common_inputs['''decoder_input_ids'''].shape[1]
       UpperCAmelCase					,		UpperCAmelCase			:				Dict										=					self.num_attention_heads
       UpperCAmelCase			:				Union[str, Any]										=					(
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       UpperCAmelCase			:				str										=					decoder_seq_length + 3
       UpperCAmelCase			:				Union[str, Any]										=					(
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )

       UpperCAmelCase			:				Union[str, Any]										=					torch.cat(
           [common_inputs['''decoder_attention_mask'''], torch.ones(__A,						__A		)],						dim=1		)

       UpperCAmelCase			:				Any										=					[]
       # If the number of encoder and decoder layers are present in the model configuration, both are considered
       UpperCAmelCase					,		UpperCAmelCase			:				str										=					self.num_layers
       UpperCAmelCase			:				Any										=					min(__A,						__A		)
       UpperCAmelCase			:				Dict										=					max(__A,						__A		) - min_num_layers
       UpperCAmelCase			:				Tuple										=					'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

       for _ in range(__A		):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(__A		),
                 torch.zeros(__A		),
                 torch.zeros(__A		),
                 torch.zeros(__A		),
             )		)
       # TODO: test this.
       UpperCAmelCase			:				List[str]										=					encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
       for _ in range(__A,						__A		):
         common_inputs["past_key_values"].append((torch.zeros(__A		), torch.zeros(__A		))		)
     return common_inputs





   UpperCamelCase  	=	_generate_dummy_inputs_for_default_and_seqaseq_lm





