






import copy
import tempfile
import unittest

from transformers import MaMaaaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
from transformers.utils import cached_property

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import MaMaaaForConditionalGeneration, MaMaaaModel, MaMaaaTokenizer
							from transformers.models.mam_aaa.modeling_mam_aaa import MaMaaaDecoder, MaMaaaEncoder






def 							A     (       a_							,a_							,a_							,a_=None							,a_=None							,a_=None							,a_=None							,a_=None							,)     ->						str:
					if attention_mask is None:
										__UpperCamelCase      :					int       		=input_ids.ne(config.pad_token_id					)
					if decoder_attention_mask is None:
										__UpperCamelCase      :					int       		=decoder_input_ids.ne(config.pad_token_id					)
					if head_mask is None:
										__UpperCamelCase      :					Tuple       		=torch.ones(config.encoder_layers							,config.encoder_attention_heads							,device=_UpperCAmelCase					)
					if decoder_head_mask is None:
										__UpperCamelCase      :					Dict       		=torch.ones(config.decoder_layers							,config.decoder_attention_heads							,device=_UpperCAmelCase					)
					if cross_attn_head_mask is None:
										__UpperCamelCase      :					Tuple       		=torch.ones(config.decoder_layers							,config.decoder_attention_heads							,device=_UpperCAmelCase					)
					return {
					    "input_ids": input_ids,
					    "decoder_input_ids": decoder_input_ids,
					    "attention_mask": attention_mask,
					    "decoder_attention_mask": attention_mask,
					    "head_mask": head_mask,
					    "decoder_head_mask": decoder_head_mask,
					    "cross_attn_head_mask": cross_attn_head_mask,
					}




class 				__A     :





				"""simple docstring"""







				def __init__( self							,  lowerCamelCase__							,  lowerCamelCase__=13							,  lowerCamelCase__=7							,  lowerCamelCase__=True							,  lowerCamelCase__=False							,  lowerCamelCase__=99							,  lowerCamelCase__=16							,  lowerCamelCase__=2							,  lowerCamelCase__=4							,  lowerCamelCase__=4							,  lowerCamelCase__="relu"							,  lowerCamelCase__=0.1							,  lowerCamelCase__=0.1							,  lowerCamelCase__=0.0							,  lowerCamelCase__=0.0							,  lowerCamelCase__=20							,  lowerCamelCase__=2							,  lowerCamelCase__=1							,  lowerCamelCase__=0							,  ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=parent
									__UpperCamelCase      :					Tuple       		=batch_size
									__UpperCamelCase      :					int       		=seq_length
									__UpperCamelCase      :					Union[str, Any]       		=is_training
									__UpperCamelCase      :					Dict       		=use_labels
									__UpperCamelCase      :					Optional[int]       		=vocab_size
									__UpperCamelCase      :					Optional[Any]       		=hidden_size
									__UpperCamelCase      :					str       		=num_hidden_layers
									__UpperCamelCase      :					Dict       		=num_attention_heads
									__UpperCamelCase      :					str       		=intermediate_size
									__UpperCamelCase      :					str       		=hidden_act
									__UpperCamelCase      :					Dict       		=hidden_dropout_prob
									__UpperCamelCase      :					Any       		=attention_probs_dropout_prob
									__UpperCamelCase      :					Dict       		=encoder_layerdrop
									__UpperCamelCase      :					List[Any]       		=decoder_layerdrop
									__UpperCamelCase      :					Any       		=max_position_embeddings
									__UpperCamelCase      :					str       		=eos_token_id
									__UpperCamelCase      :					int       		=pad_token_id
									__UpperCamelCase      :					Union[str, Any]       		=bos_token_id


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=ids_tensor([self.batch_size, self.seq_length]							,  self.vocab_size    )
									__UpperCamelCase      :					str       		=self.eos_token_id  # Eos Token
									__UpperCamelCase      :					int       		=ids_tensor([self.batch_size, self.seq_length]							,  self.vocab_size    )

									# we need to clamp the input ids here to avoid having pad token in between
									# this is because for M2M100 the position_ids are prepared such that
									# all pad tokens have pos id = 2 and rest are between 2..seq_length
									# and the seq_length here is seq_length - num_pad_tokens
									# but when using past, there is no way of knowing if the past input ids had
									# pad tokens in them, which results in incorrect seq_lenth and which in turn results in
									# position_ids being off by num_pad_tokens in past input
									__UpperCamelCase      :					Any       		=input_ids.clamp(self.pad_token_id + 1    )
									__UpperCamelCase      :					Dict       		=decoder_input_ids.clamp(self.pad_token_id + 1    )

									__UpperCamelCase      :					Tuple       		=self.get_config()
									__UpperCamelCase      :					str       		=prepare_mam_aaa_inputs_dict(lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    )
									return config, inputs_dict


				def       __lowercase							( self    ):




									"""simple docstring"""

									return MaMaaaConfig(
									    vocab_size=self.vocab_size							,  d_model=self.hidden_size							,  encoder_layers=self.num_hidden_layers							,  decoder_layers=self.num_hidden_layers							,  encoder_attention_heads=self.num_attention_heads							,  decoder_attention_heads=self.num_attention_heads							,  encoder_ffn_dim=self.intermediate_size							,  decoder_ffn_dim=self.intermediate_size							,  dropout=self.hidden_dropout_prob							,  attention_dropout=self.attention_probs_dropout_prob							,  encoder_layerdrop=self.encoder_layerdrop							,  decoder_layerdrop=self.decoder_layerdrop							,  max_position_embeddings=self.max_position_embeddings							,  eos_token_id=self.eos_token_id							,  bos_token_id=self.bos_token_id							,  pad_token_id=self.pad_token_id							,  )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=self.prepare_config_and_inputs()
									return config, inputs_dict


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=MaMaaaModel(config=lowerCamelCase__    ).get_decoder().to(lowerCamelCase__    ).eval()
									__UpperCamelCase      :					Tuple       		=inputs_dict['input_ids']
									__UpperCamelCase      :					Tuple       		=inputs_dict['attention_mask']
									__UpperCamelCase      :					str       		=inputs_dict['head_mask']

									# first forward pass
									__UpperCamelCase      :					Dict       		=model(lowerCamelCase__							,  attention_mask=lowerCamelCase__							,  head_mask=lowerCamelCase__							,  use_cache=lowerCamelCase__    )

									__UpperCamelCase      :					Optional[int]       		=outputs.to_tuple()

									# create hypothetical multiple next token and extent to next_input_ids
									__UpperCamelCase      :					Tuple       		=ids_tensor((self.batch_size, 3)							,  config.vocab_size    )
									__UpperCamelCase      :					Any       		=ids_tensor((self.batch_size, 3)							,  2    )

									# append to next input_ids and
									__UpperCamelCase      :					Optional[int]       		=torch.cat([input_ids, next_tokens]							,  dim=-1    )
									__UpperCamelCase      :					Optional[Any]       		=torch.cat([attention_mask, next_attn_mask]							,  dim=-1    )

									__UpperCamelCase      :					Tuple       		=model(lowerCamelCase__							,  attention_mask=lowerCamelCase__    )['last_hidden_state']
									__UpperCamelCase      :					str       		=model(lowerCamelCase__							,  attention_mask=lowerCamelCase__							,  past_key_values=lowerCamelCase__    )[
									    'last_hidden_state'
									]

									# select random slice
									__UpperCamelCase      :					int       		=ids_tensor((1,)							,  output_from_past.shape[-1]    ).item()
									__UpperCamelCase      :					str       		=output_from_no_past[:, -3:, random_slice_idx].detach()
									__UpperCamelCase      :					Any       		=output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(lowerCamelCase__							,  lowerCamelCase__							,  atol=1E-2    )    )







				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=MaMaaaModel(config=lowerCamelCase__    ).to(lowerCamelCase__    ).eval()
									__UpperCamelCase      :					Optional[int]       		=model(**lowerCamelCase__    )

									__UpperCamelCase      :					Dict       		=outputs.encoder_last_hidden_state
									__UpperCamelCase      :					int       		=outputs.last_hidden_state

									with tempfile.TemporaryDirectory() as tmpdirname:
														__UpperCamelCase      :					Optional[int]       		=model.get_encoder()
														encoder.save_pretrained(lowerCamelCase__    )
														__UpperCamelCase      :					List[Any]       		=MaMaaaEncoder.from_pretrained(lowerCamelCase__    ).to(lowerCamelCase__    )

									__UpperCamelCase      :					int       		=encoder(inputs_dict['input_ids']							,  attention_mask=inputs_dict['attention_mask']    )[
									    0
									]

									self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1E-3    )

									with tempfile.TemporaryDirectory() as tmpdirname:
														__UpperCamelCase      :					List[Any]       		=model.get_decoder()
														decoder.save_pretrained(lowerCamelCase__    )
														__UpperCamelCase      :					Any       		=MaMaaaDecoder.from_pretrained(lowerCamelCase__    ).to(lowerCamelCase__    )

									__UpperCamelCase      :					Any       		=decoder(
									    input_ids=inputs_dict['decoder_input_ids']							,  attention_mask=inputs_dict['decoder_attention_mask']							,  encoder_hidden_states=lowerCamelCase__							,  encoder_attention_mask=inputs_dict['attention_mask']							,  )[0]

									self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1E-3    )




@require_torch
class 				__A     (			__UpperCamelCase      ,     __UpperCamelCase      ,     __UpperCamelCase      ,     unittest.TestCase ):





				"""simple docstring"""







				UpperCamelCase__ :      Any			 =(
				    (
				        MaMaaaModel,
				        MaMaaaForConditionalGeneration,
				    )
				    if is_torch_available()
				    else ()
				)
				UpperCamelCase__ :      Optional[int]			 =(MaMaaaForConditionalGeneration,) if is_torch_available() else ()
				UpperCamelCase__ :      Union[str, Any]			 =(
				    {
				        """conversational""": MaMaaaForConditionalGeneration,
				        """feature-extraction""": MaMaaaModel,
				        """summarization""": MaMaaaForConditionalGeneration,
				        """text2text-generation""": MaMaaaForConditionalGeneration,
				        """translation""": MaMaaaForConditionalGeneration,
				    }
				    if is_torch_available()
				    else {}
				)
				UpperCamelCase__ :      Tuple			 =True
				UpperCamelCase__ :      Any			 =True
				UpperCamelCase__ :      str			 =False
				UpperCamelCase__ :      Optional[Any]			 =False


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__							,  lowerCamelCase__    ):




									"""simple docstring"""

									if pipeline_test_casse_name == "TranslationPipelineTests":
														# Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`.
														# `M2M100Config` was never used in pipeline tests: cannot create a simple tokenizer.
														return True

									return False


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=MaMaaaModelTester(self    )
									__UpperCamelCase      :					Dict       		=ConfigTester(self							,  config_class=lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									self.config_tester.run_common_tests()


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
														__UpperCamelCase      :					Any       		=model_class(lowerCamelCase__    )

														with tempfile.TemporaryDirectory() as tmpdirname:
																			model.save_pretrained(lowerCamelCase__    )
																			__UpperCamelCase      :					List[Any]       		=model_class.from_pretrained(lowerCamelCase__							,  output_loading_info=lowerCamelCase__    )
														self.assertEqual(info['missing_keys']							,  []    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					str       		=self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_encoder_decoder_model_standalone(*lowerCamelCase__    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in (MaMaaaModel, MaMaaaForConditionalGeneration):
														__UpperCamelCase      :					Union[str, Any]       		=model_class(lowerCamelCase__    )
														model.to(lowerCamelCase__    )
														model.eval()

														__UpperCamelCase      :					List[str]       		=copy.deepcopy(self._prepare_for_class(lowerCamelCase__							,  lowerCamelCase__    )    )

														if not self.is_encoder_decoder:
																			__UpperCamelCase      :					Tuple       		=inputs['input_ids']
																			del inputs["input_ids"]
														else:
																			__UpperCamelCase      :					List[str]       		=inputs['input_ids']
																			__UpperCamelCase      :					List[Any]       		=inputs.get('decoder_input_ids'							,  lowerCamelCase__    )
																			del inputs["input_ids"]
																			inputs.pop('decoder_input_ids'							,  lowerCamelCase__    )

														__UpperCamelCase      :					List[str]       		=model.get_input_embeddings()
														if not self.is_encoder_decoder:
																			__UpperCamelCase      :					List[Any]       		=wte(lowerCamelCase__    )
														else:
																			__UpperCamelCase      :					int       		=wte(lowerCamelCase__    )
																			__UpperCamelCase      :					Optional[Any]       		=wte(lowerCamelCase__    )

														with torch.no_grad():
																			model(**lowerCamelCase__    )[0]







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Tuple       		=self.model_tester.prepare_config_and_inputs()
									__UpperCamelCase      :					Dict       		=input_dict['input_ids']
									__UpperCamelCase      :					List[str]       		=input_ids.ne(1    ).to(lowerCamelCase__    )
									__UpperCamelCase      :					List[str]       		=MaMaaaForConditionalGeneration(lowerCamelCase__    ).eval().to(lowerCamelCase__    )
									if torch_device == "cuda":
														model.half()
									model.generate(lowerCamelCase__							,  attention_mask=lowerCamelCase__    )
									model.generate(num_beams=4							,  do_sample=lowerCamelCase__							,  early_stopping=lowerCamelCase__							,  num_return_sequences=3    )






def 							A     (       a_					)     ->						Optional[Any]:
					return torch.tensor(_UpperCAmelCase							,dtype=torch.long							,device=_UpperCAmelCase					)


A_       :Union[str, Any]       =						1E-4




@require_torch
@require_sentencepiece
@require_tokenizers
@slow
class 				__A     (			unittest.TestCase ):





				"""simple docstring"""







				@cached_property
				def       __lowercase							( self    ):




									"""simple docstring"""

									return MaMaaaTokenizer.from_pretrained('facebook/m2m100_418M'    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=MaMaaaModel.from_pretrained('facebook/m2m100_418M'    ).to(lowerCamelCase__    )
									__UpperCamelCase      :					str       		=_long_tensor([[128028, 98, 12, 30527, 2732, 159, 7755, 61904, 39144, 38, 2]]    )
									__UpperCamelCase      :					Any       		=_long_tensor([[2, 128028, 98, 12, 30527, 2732, 159, 7755, 61904, 39144, 38]]    )
									__UpperCamelCase      :					Optional[int]       		=prepare_mam_aaa_inputs_dict(model.config							,  lowerCamelCase__							,  lowerCamelCase__    )
									with torch.no_grad():
														__UpperCamelCase      :					Tuple       		=model(**lowerCamelCase__    )[0]
									__UpperCamelCase      :					Tuple       		=torch.Size((1, 11, 1024)    )
									self.assertEqual(output.shape							,  lowerCamelCase__    )
									# change to expected output here
									__UpperCamelCase      :					int       		=torch.tensor(
									    [[-0.7_780, -0.1_676, 0.1_038], [-6.7_556, -1.3_992, 0.0_567], [-7.5_383, -0.5_920, -0.2_779]]							,  device=lowerCamelCase__    )
									self.assertTrue(torch.allclose(output[:, :3, :3]							,  lowerCamelCase__							,  atol=lowerCamelCase__    )    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=MaMaaaForConditionalGeneration.from_pretrained('facebook/m2m100_418M'    ).to(lowerCamelCase__    )

									# change to intended input
									__UpperCamelCase      :					Union[str, Any]       		=_long_tensor([[128028, 98, 12, 30527, 2732, 159, 7755, 61904, 39144, 38, 2]]    )
									__UpperCamelCase      :					Tuple       		=_long_tensor([[2, 128028, 98, 12, 30527, 2732, 159, 7755, 61904, 39144, 38]]    )
									__UpperCamelCase      :					Optional[int]       		=prepare_mam_aaa_inputs_dict(model.config							,  lowerCamelCase__							,  lowerCamelCase__    )
									with torch.no_grad():
														__UpperCamelCase      :					str       		=model(**lowerCamelCase__    )[0]
									__UpperCamelCase      :					Dict       		=torch.Size((1, 11, model.config.vocab_size)    )
									self.assertEqual(output.shape							,  lowerCamelCase__    )
									# change to expected output here
									__UpperCamelCase      :					Optional[int]       		=torch.tensor(
									    [[-1.0_448, -1.0_411, 3.7_992], [-3.2_191, -3.2_386, -1.3_451], [-3.6_210, -3.5_993, 0.4_925]]							,  device=lowerCamelCase__    )
									self.assertTrue(torch.allclose(output[:, :3, :3]							,  lowerCamelCase__							,  atol=lowerCamelCase__    )    )







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=MaMaaaForConditionalGeneration.from_pretrained('facebook/m2m100_418M'    ).to(lowerCamelCase__    )
									__UpperCamelCase      :					str       		=MaMaaaTokenizer.from_pretrained('facebook/m2m100_418M'							,  src_lang='fr'							,  tgt_lang='en'    )

									__UpperCamelCase      :					Dict       		=[
									    'L\'affaire NSA souligne l\'absence totale de débat sur le renseignement',
									    'Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.',
									    'Lorsque François Hollande téléphone à Barack Obama ou quand le ministre des affaires étrangères Laurent'
									    ' Fabius convoque l\'ambassadeur des Etats-Unis, ils réagissent à une vraie découverte, qui est celle de'
									    ' l\'ampleur de la surveillance américaine sur l\'ensemble des communications en France.',
									]

									# The below article tests that we don't add any hypotheses outside of the top n_beams
									__UpperCamelCase      :					List[str]       		=tokenizer(lowerCamelCase__							,  padding=lowerCamelCase__							,  return_tensors='pt'    )

									__UpperCamelCase      :					Tuple       		=model.generate(
									    input_ids=dct['input_ids'].to(lowerCamelCase__    )							,  attention_mask=dct['attention_mask'].to(lowerCamelCase__    )							,  num_beams=5							,  forced_bos_token_id=tokenizer.get_lang_id('en'    )							,  )

									__UpperCamelCase      :					str       		=[
									    'The NSA case highlights the total absence of intelligence debate',
									    'I think there are two levels of response from the French government.',
									    'When François Hollande calls Barack Obama or when Foreign Minister Laurent Fabius calls the U.S.'
									    ' Ambassador, they respond to a real discovery, which is that of the scale of U.S. surveillance on all'
									    ' communications in France.',
									]

									__UpperCamelCase      :					Tuple       		=tokenizer.batch_decode(
									    hypotheses_batch.tolist()							,  clean_up_tokenization_spaces=lowerCamelCase__							,  skip_special_tokens=lowerCamelCase__    )
									assert generated == expected_en








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase				:       Optional[int]	   =				logging.get_logger(__name__)

_UpperCAmelCase				:       Any	   =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       int	   =				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       int	   =				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase				:       Any	   =				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		RealmTokenizer
 def __init__(      self  :      Optional[int]			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      List[Any]=True			,     UpperCAmelCase  :      Optional[Any]="[UNK]"			,     UpperCAmelCase  :      Any="[SEP]"			,     UpperCAmelCase  :      Tuple="[PAD]"			,     UpperCAmelCase  :      List[Any]="[CLS]"			,     UpperCAmelCase  :      Union[str, Any]="[MASK]"			,     UpperCAmelCase  :      Optional[Any]=True			,     UpperCAmelCase  :      Any=None			,     **UpperCAmelCase  :      Optional[int]			,     )			->	str:
     super().__init__(
         UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     do_lower_case=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     tokenize_chinese_chars=UpperCAmelCase			,     strip_accents=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
     if (
         normalizer_state.get('lowercase'			,     UpperCAmelCase	) != do_lower_case
         or normalizer_state.get('strip_accents'			,     UpperCAmelCase	) != strip_accents
         or normalizer_state.get('handle_chinese_chars'			,     UpperCAmelCase	) != tokenize_chinese_chars
     ):
         lowerCamelCase__    :      Optional[int]					 =		getattr(UpperCAmelCase			,     normalizer_state.pop('type'	)	)
         lowerCamelCase__    :      Optional[Any]					 =		do_lower_case
         lowerCamelCase__    :      str					 =		strip_accents
         lowerCamelCase__    :      Optional[Any]					 =		tokenize_chinese_chars
         lowerCamelCase__    :      int					 =		normalizer_class(**UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		do_lower_case
 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      int			,     **UpperCAmelCase  :      int	)			->	List[Any]:
     lowerCamelCase__    :      List[Any]					 =		PaddingStrategy.MAX_LENGTH

     lowerCamelCase__    :      Optional[int]					 =		text
     lowerCamelCase__    :      Dict					 =		kwargs.pop('text_pair'			,     UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		kwargs.pop('return_tensors'			,     UpperCAmelCase	)

     lowerCamelCase__    :      List[Any]					 =		{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase	):
         if batch_text_pair is not None:
             lowerCamelCase__    :      Tuple					 =		batch_text_pair[idx]
         else:
             lowerCamelCase__    :      Dict					 =		None

         lowerCamelCase__    :      Optional[int]					 =		super().__call__(UpperCAmelCase			,     UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)

         lowerCamelCase__    :      Any					 =		encoded_candidates.get('input_ids'	)
         lowerCamelCase__    :      Union[str, Any]					 =		encoded_candidates.get('attention_mask'	)
         lowerCamelCase__    :      Tuple					 =		encoded_candidates.get('token_type_ids'	)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase	)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase	)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		{key: item for key, item in output_data.items() if len(UpperCAmelCase	) != 0}

     return BatchEncoding(UpperCAmelCase			,     tensor_type=UpperCAmelCase	)
 def      A_	(      self  :      int			,     UpperCAmelCase  :      Optional[Any]			,     UpperCAmelCase  :      Optional[int]=None	)			->	List[str]:
     lowerCamelCase__    :      Tuple					 =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      List[Any]					 =		[self.sep_token_id]
     lowerCamelCase__    :      Optional[int]					 =		[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     lowerCamelCase__    :      int					 =		self._tokenizer.model.save(UpperCAmelCase			,     name=UpperCAmelCase	)
     return tuple(UpperCAmelCase	)





