






from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


A										=      logging.get_logger(__name__)

A										=      {
    'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}


class 				__a				(	__A    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Union[str, Any]					     =    """m2m_100"""
		UpperCAmelCase__   :					str					     =    ["""past_key_values"""]
		UpperCAmelCase__   :					Union[str, Any]					     =    {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}


		def __init__(    self			,     UpperCamelCase__=128112			,     UpperCamelCase__=1024			,     UpperCamelCase__=12			,     UpperCamelCase__=4096			,     UpperCamelCase__=16			,     UpperCamelCase__=12			,     UpperCamelCase__=4096			,     UpperCamelCase__=16			,     UpperCamelCase__=0.05			,     UpperCamelCase__=0.05			,     UpperCamelCase__=True			,     UpperCamelCase__=True			,     UpperCamelCase__="relu"			,     UpperCamelCase__=1024			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.1			,     UpperCamelCase__=0.0			,     UpperCamelCase__=0.02			,     UpperCamelCase__=2			,     UpperCamelCase__=True			,     UpperCamelCase__=1			,     UpperCamelCase__=0			,     UpperCamelCase__=2			,     **UpperCamelCase__			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= vocab_size
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= max_position_embeddings
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= d_model
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= encoder_ffn_dim
						SCREAMING_SNAKE_CASE_ :						List[str]    				= encoder_layers
						SCREAMING_SNAKE_CASE_ :						Any    				= encoder_attention_heads
						SCREAMING_SNAKE_CASE_ :						int    				= decoder_ffn_dim
						SCREAMING_SNAKE_CASE_ :						str    				= decoder_layers
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= decoder_attention_heads
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= dropout
						SCREAMING_SNAKE_CASE_ :						int    				= attention_dropout
						SCREAMING_SNAKE_CASE_ :						Dict    				= activation_dropout
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= activation_function
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= init_std
						SCREAMING_SNAKE_CASE_ :						Dict    				= encoder_layerdrop
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= decoder_layerdrop
						SCREAMING_SNAKE_CASE_ :						Dict    				= use_cache
						SCREAMING_SNAKE_CASE_ :						Dict    				= encoder_layers
						SCREAMING_SNAKE_CASE_ :						Tuple    				= scale_embedding  # scale factor will be sqrt(d_model) if True

						super().__init__(
						    pad_token_id=UpperCamelCase__			,     bos_token_id=UpperCamelCase__			,     eos_token_id=UpperCamelCase__			,     is_encoder_decoder=UpperCamelCase__			,     decoder_start_token_id=UpperCamelCase__			,     **UpperCamelCase__			,     )






class 				__a				(	__A    ):





		'''simple docstring'''



		@property
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						int    				= OrderedDict(
						    [
						        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
						        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
						    ]							)

						if self.use_past:
										SCREAMING_SNAKE_CASE_ :						Dict    				= {0: 'batch'}
										SCREAMING_SNAKE_CASE_ :						List[Any]    				= {0: 'batch', 1: 'past_decoder_sequence + sequence'}
						else:
										SCREAMING_SNAKE_CASE_ :						Tuple    				= {0: 'batch', 1: 'decoder_sequence'}
										SCREAMING_SNAKE_CASE_ :						Optional[int]    				= {0: 'batch', 1: 'decoder_sequence'}

						if self.use_past:
										self.fill_with_past_key_values_(UpperCamelCase__			,     direction='inputs'							)
						return common_inputs


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__ = -1			,     UpperCamelCase__ = -1			,     UpperCamelCase__ = False			,     UpperCamelCase__ = None			,     ):
						# Copied from OnnxConfig.generate_dummy_inputs
						# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
						# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= compute_effective_axis_dimension(
						    UpperCamelCase__			,     fixed_dimension=OnnxConfig.default_fixed_batch			,     num_token_to_add=0							)

						# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= tokenizer.num_special_tokens_to_add(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Any    				= compute_effective_axis_dimension(
						    UpperCamelCase__			,     fixed_dimension=OnnxConfig.default_fixed_sequence			,     num_token_to_add=UpperCamelCase__							)

						# Generate dummy inputs according to compute batch and sequence
						SCREAMING_SNAKE_CASE_ :						List[str]    				= [' '.join([tokenizer.unk_token]							) * seq_length] * batch_size
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= dict(tokenizer(UpperCamelCase__			,     return_tensors=UpperCamelCase__							)							)
						return common_inputs


		def 	__snake_case      (    self			,     UpperCamelCase__			,     UpperCamelCase__ = -1			,     UpperCamelCase__ = -1			,     UpperCamelCase__ = False			,     UpperCamelCase__ = None			,     ):
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							)

						# Generate decoder inputs
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= seq_length if not self.use_past else 1
						SCREAMING_SNAKE_CASE_ :						Dict    				= self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__			,     UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						int    				= {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
						SCREAMING_SNAKE_CASE_ :						str    				= dict(**UpperCamelCase__			,     **UpperCamelCase__							)

						if self.use_past:
										if not is_torch_available():
														raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'							)
										else:
														import torch
										SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_ :						Tuple    				= common_inputs['input_ids'].shape
										SCREAMING_SNAKE_CASE_ :						Optional[int]    				= common_inputs['decoder_input_ids'].shape[1]
										SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.num_attention_heads
										SCREAMING_SNAKE_CASE_ :						str    				= (
										    batch,
										    num_encoder_attention_heads,
										    encoder_seq_length,
										    self._config.hidden_size // num_encoder_attention_heads,
										)
										SCREAMING_SNAKE_CASE_ :						List[Any]    				= decoder_seq_length + 3
										SCREAMING_SNAKE_CASE_ :						int    				= (
										    batch,
										    num_decoder_attention_heads,
										    decoder_past_length,
										    self._config.hidden_size // num_decoder_attention_heads,
										)

										SCREAMING_SNAKE_CASE_ :						Tuple    				= torch.cat(
										    [common_inputs['decoder_attention_mask'], torch.ones(UpperCamelCase__			,     UpperCamelCase__							)]			,     dim=1							)

										SCREAMING_SNAKE_CASE_ :						Optional[int]    				= []
										# If the number of encoder and decoder layers are present in the model configuration, both are considered
										SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_ :						List[Any]    				= self.num_layers
										SCREAMING_SNAKE_CASE_ :						Any    				= min(UpperCamelCase__			,     UpperCamelCase__							)
										SCREAMING_SNAKE_CASE_ :						List[str]    				= max(UpperCamelCase__			,     UpperCamelCase__							) - min_num_layers
										SCREAMING_SNAKE_CASE_ :						List[Any]    				= 'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

										for _ in range(UpperCamelCase__							):
														common_inputs["past_key_values"].append(
														    (
														        torch.zeros(UpperCamelCase__							),
														        torch.zeros(UpperCamelCase__							),
														        torch.zeros(UpperCamelCase__							),
														        torch.zeros(UpperCamelCase__							),
														    )							)
										# TODO: test this.
										SCREAMING_SNAKE_CASE_ :						str    				= encoder_shape if remaining_side_name == 'encoder' else decoder_shape
										for _ in range(UpperCamelCase__			,     UpperCamelCase__							):
														common_inputs["past_key_values"].append((torch.zeros(UpperCamelCase__							), torch.zeros(UpperCamelCase__							))							)
						return common_inputs






		UpperCAmelCase__   :					List[str]					     =    _generate_dummy_inputs_for_default_and_seqaseq_lm







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 				__a				(	__A			,						unittest.TestCase    ):





		'''simple docstring'''



		UpperCAmelCase__   :					Optional[Any]					     =    HerbertTokenizer
		UpperCAmelCase__   :					List[Any]					     =    HerbertTokenizerFast
		UpperCAmelCase__   :					Optional[Any]					     =    True


		def 	__snake_case      (    self							):
						super().setUp()

						# Use a simpler test file without japanese/chinese characters
						with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''			,     encoding='utf-8'							) as f_data:
										SCREAMING_SNAKE_CASE_ :						Dict    				= f_data.read().replace('\n\n'			,     '\n'							).strip()

						SCREAMING_SNAKE_CASE_ :						str    				= [
						    '<s>',
						    '</s>',
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    'w</w>',
						    'r</w>',
						    't</w>',
						    'lo',
						    'low',
						    'er</w>',
						    'low</w>',
						    'lowest</w>',
						    'newer</w>',
						    'wider</w>',
						    ',</w>',
						    '<unk>',
						]
						SCREAMING_SNAKE_CASE_ :						str    				= dict(zip(UpperCamelCase__			,     range(len(UpperCamelCase__							)							)							)							)
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

						SCREAMING_SNAKE_CASE_ :						int    				= os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['vocab_file']							)
						SCREAMING_SNAKE_CASE_ :						int    				= os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['merges_file']							)
						with open(self.vocab_file			,     'w'							) as fp:
										fp.write(json.dumps(UpperCamelCase__							)							)
						with open(self.merges_file			,     'w'							) as fp:
										fp.write('\n'.join(UpperCamelCase__							)							)


		def 	__snake_case      (    self			,     UpperCamelCase__							):
						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= 'lower newer'
						SCREAMING_SNAKE_CASE_ :						int    				= 'lower newer'
						return input_text, output_text


		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= self.tokenizer_class(vocab_file=self.vocab_file			,     merges_file=self.merges_file							)

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= 'lower'
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= ['low', 'er</w>']
						SCREAMING_SNAKE_CASE_ :						int    				= tokenizer.tokenize(UpperCamelCase__							)
						self.assertListEqual(UpperCamelCase__			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= tokens + ['<unk>']
						SCREAMING_SNAKE_CASE_ :						List[Any]    				= [16, 17, 23]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__							)			,     UpperCamelCase__							)


		def 	__snake_case      (    self							):
						if not self.test_rust_tokenizer:
										return

						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.get_tokenizer()
						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= self.get_rust_tokenizer()

						SCREAMING_SNAKE_CASE_ :						List[str]    				= 'lower,newer'

						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= tokenizer.tokenize(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Tuple    				= rust_tokenizer.tokenize(UpperCamelCase__							)
						self.assertListEqual(UpperCamelCase__			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Optional[Any]    				= tokenizer.encode(UpperCamelCase__			,     add_special_tokens=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Any    				= rust_tokenizer.encode(UpperCamelCase__			,     add_special_tokens=UpperCamelCase__							)
						self.assertListEqual(UpperCamelCase__			,     UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Union[str, Any]    				= self.get_rust_tokenizer()
						SCREAMING_SNAKE_CASE_ :						Dict    				= tokenizer.encode(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= rust_tokenizer.encode(UpperCamelCase__							)
						self.assertListEqual(UpperCamelCase__			,     UpperCamelCase__							)






		@slow
		def 	__snake_case      (    self							):
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'							)

						SCREAMING_SNAKE_CASE_ :						List[str]    				= tokenizer.encode('konstruowanie sekwencji'			,     add_special_tokens=UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Optional[int]    				= tokenizer.encode('konstruowanie wielu sekwencji'			,     add_special_tokens=UpperCamelCase__							)

						SCREAMING_SNAKE_CASE_ :						Any    				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__							)
						SCREAMING_SNAKE_CASE_ :						Dict    				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__			,     UpperCamelCase__							)

						assert encoded_sentence == [0] + text + [2]
						assert encoded_pair == [0] + text + [2] + text_a + [2]