






"""simple docstring"""






import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
  from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
  UpperCAmelCase_   : str   			= get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase_   : str   			= 128022
UpperCAmelCase_   : Optional[Any]   			= 128028




@require_sentencepiece
class 							lowerCAmelCase__				(			SCREAMING_SNAKE_CASE_				,		unittest.TestCase ):






      '''simple docstring'''

      __UpperCamelCase	 =  MaMaaaTokenizer
      __UpperCamelCase	 =  False
      __UpperCamelCase	 =  False
      __UpperCamelCase	 =  True
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




          '''simple docstring'''


          super().setUp()

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
          SCREAMING_SNAKE_CASE_      :  Dict    						=		dict(zip(a__						,	range(len(a__))))
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		Path(self.tmpdirname)
          save_json(a__						,	save_dir / VOCAB_FILES_NAMES['''vocab_file'''])
          if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
              copyfile(a__						,	save_dir / VOCAB_FILES_NAMES['''spm_file'''])

          SCREAMING_SNAKE_CASE_      :  Any    						=		MaMaaaTokenizer.from_pretrained(self.tmpdirname)
          tokenizer.save_pretrained(self.tmpdirname)
      def _SCREAMING_SNAKE_CASE (						self     :  str						,	**lowercase_     :  List[str]):




          '''simple docstring'''


          return MaMaaaTokenizer.from_pretrained(self.tmpdirname						,	**a__)
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]						,	lowercase_     :  Optional[int]):




          '''simple docstring'''


          return (
              "This is a test",
              "This is a test",
          )
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		"""</s>"""
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__)						,	a__)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__)						,	a__)
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		self.get_tokenizer()
          SCREAMING_SNAKE_CASE_      :  Any    						=		list(tokenizer.get_vocab().keys())

          self.assertEqual(vocab_keys[0]						,	'''</s>''')
          self.assertEqual(vocab_keys[1]						,	'''<unk>''')
          self.assertEqual(vocab_keys[-1]						,	'''<s>''')
          self.assertEqual(len(a__)						,	tokenizer.vocab_size + len(tokenizer.get_added_vocab()))
      @unittest.skip('''Skip this test while all models are still to be uploaded.''')
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          pass
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :  Tuple    						=		tokenizer.tokenize('''This is a test''')
          self.assertListEqual(a__						,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(a__)						,	[2, 3, 4, 5, 6]						,	)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6])
          self.assertListEqual(a__						,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

          SCREAMING_SNAKE_CASE_      :  int    						=		tokenizer.convert_tokens_to_string(a__)
          self.assertEqual(a__						,	'''This is a test''')



      @slow
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		{"""input_ids""": [[128022, 110108, 397, 11, 38272, 2247, 124811, 285, 18105, 1586, 207, 7, 39534, 4428, 397, 1019, 18105, 1586, 207, 7, 41337, 16786, 241, 7, 20214, 17, 125690, 10398, 7, 44378, 58069, 68342, 7798, 7343, 11, 299, 33310, 4, 158, 37350, 94077, 4569, 299, 33310, 90, 4, 52840, 290, 4, 31270, 112, 299, 682, 4, 52840, 39953, 14079, 193, 52519, 90894, 17894, 120697, 11, 40445, 551, 17, 1019, 52519, 90894, 17756, 963, 11, 40445, 480, 17, 9792, 1120, 5173, 1393, 6240, 16786, 241, 120996, 28, 1245, 1393, 118240, 11123, 1019, 93612, 2691, 10618, 98058, 120409, 1928, 279, 4, 40683, 367, 178, 207, 1019, 103, 103121, 506, 65296, 5, 2], [128022, 21217, 367, 117, 125450, 128, 719, 7, 7308, 40, 93612, 12669, 1116, 16704, 71, 17785, 3699, 15592, 35, 144, 9584, 241, 11943, 713, 950, 799, 2247, 88427, 150, 149, 118813, 120706, 1019, 106906, 81518, 28, 1224, 22799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128022, 1658, 123311, 5155, 5578, 4722, 279, 14947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=a__						,	model_name='''facebook/m2m100_418M'''						,	revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''						,	)




@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      __UpperCamelCase	 =  "facebook/m2m100_418M"
      __UpperCamelCase	 =  [
          "In my opinion, there are two levels of response from the French government.",
          "NSA Affair Emphasizes Complete Lack of Debate on Intelligence",
      ]
      __UpperCamelCase	 =  [
          "Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.",
          "L'affaire NSA souligne l'absence totale de débat sur le renseignement",
      ]

      # fmt: off
      __UpperCamelCase	 =  [EN_CODE, 5_9_3, 1_9_4_9, 1_1_5_7_8_1, 4, 7_1_5_8_6, 4_2_3_4, 6_0_6_3_3, 1_2_6_2_3_3, 4_3_2, 1_2_3_8_0_8, 1_5_5_9_2, 1_1_9_7, 1_1_7_1_3_2, 1_2_0_6_1_8, 5, 2]
      @classmethod
      def _SCREAMING_SNAKE_CASE (						cls     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  MaMaaaTokenizer    						=		MaMaaaTokenizer.from_pretrained(
              cls.checkpoint_name						,	src_lang='''en'''						,	tgt_lang='''fr''')
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		1
          return cls
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          self.assertEqual(self.tokenizer.get_lang_id('''ar''')						,	128006)
          self.assertEqual(self.tokenizer.get_lang_id('''en''')						,	128022)
          self.assertEqual(self.tokenizer.get_lang_id('''ro''')						,	128076)
          self.assertEqual(self.tokenizer.get_lang_id('''mr''')						,	128063)
      def _SCREAMING_SNAKE_CASE (						self     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		self.tokenizer.get_vocab()
          self.assertEqual(len(a__)						,	self.tokenizer.vocab_size)
          self.assertEqual(vocab['''<unk>''']						,	3)
          self.assertIn(self.tokenizer.get_lang_token('''en''')						,	a__)
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		"""en"""
          SCREAMING_SNAKE_CASE_      :  int    						=		self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
          self.assertListEqual(self.expected_src_tokens						,	a__)
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          self.assertIn(a__						,	self.tokenizer.all_special_ids)
          # fmt: off
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		[FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 14028, 136, 3286, 9706, 6, 90797, 6, 144012, 162, 88128, 30061, 5, 2]
          # fmt: on
          SCREAMING_SNAKE_CASE_      :  str    						=		self.tokenizer.decode(a__						,	skip_special_tokens=a__)
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		self.tokenizer.decode(generated_ids[1:]						,	skip_special_tokens=a__)
          self.assertEqual(a__						,	a__)
          self.assertNotIn(self.tokenizer.eos_token						,	a__)
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		tempfile.mkdtemp()
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		self.tokenizer.lang_token_to_id
          self.tokenizer.save_pretrained(a__)
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		MaMaaaTokenizer.from_pretrained(a__)
          self.assertDictEqual(new_tok.lang_token_to_id						,	a__)
      @require_torch
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		"""en"""
          SCREAMING_SNAKE_CASE_      :  int    						=		"""fr"""

          SCREAMING_SNAKE_CASE_      :  Any    						=		self.tokenizer(self.src_text						,	text_target=self.tgt_text						,	padding=a__						,	return_tensors='''pt''')

          SCREAMING_SNAKE_CASE_      :  int    						=		shift_tokens_right(
              batch['''labels''']						,	self.tokenizer.pad_token_id						,	self.tokenizer.eos_token_id)

          for k in batch:
              SCREAMING_SNAKE_CASE_      :  Any    						=		batch[k].tolist()
          # batch = {k: v.tolist() for k,v in batch.items()}
          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          # batch.decoder_inputs_ids[0][0] ==
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == FR_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]
      @require_torch
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		"""mr"""
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id('''mr''')])
          self.assertListEqual(self.tokenizer.suffix_tokens						,	[self.tokenizer.eos_token_id])

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		"""zh"""
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id('''zh''')])
          self.assertListEqual(self.tokenizer.suffix_tokens						,	[self.tokenizer.eos_token_id])
      @require_torch
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		"""mr"""
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id('''mr''')])
          self.assertListEqual(self.tokenizer.suffix_tokens						,	[self.tokenizer.eos_token_id])
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang)])

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		"""zh"""
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id('''zh''')])
          self.assertListEqual(self.tokenizer.suffix_tokens						,	[self.tokenizer.eos_token_id])
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens						,	[self.tokenizer.get_lang_id(self.tokenizer.src_lang)])



      @require_torch
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		self.tokenizer._build_translation_inputs('''A test'''						,	return_tensors='''pt'''						,	src_lang='''en'''						,	tgt_lang='''ar''')

          self.assertEqual(
              nested_simplify(a__)						,	{
                  # en_XX, A, test, EOS
                  '''input_ids''': [[128022, 58, 4183, 2]],
                  '''attention_mask''': [[1, 1, 1, 1]],
                  # ar_AR
                  '''forced_bos_token_id''': 128006,
              }						,	)



"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_a      :		List[Any]           =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a      :		Union[str, Any]           =   {
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

_a      :		Optional[Any]           =   {
    'google/electra-small-generator': 512,
    'google/electra-base-generator': 512,
    'google/electra-large-generator': 512,
    'google/electra-small-discriminator': 512,
    'google/electra-base-discriminator': 512,
    'google/electra-large-discriminator': 512,
}

_a      :		Any           =   {
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}
class 						__A							( SCREAMING_SNAKE_CASE_		):
     _UpperCamelCase							:			Tuple							=		VOCAB_FILES_NAMES
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_INIT_CONFIGURATION
     _UpperCamelCase							:			Union[str, Any]							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:			Optional[Any]							=		ElectraTokenizer






     def __init__(		self				,     a__=None				,     a__=None				,     a__=True				,     a__="[UNK]"				,     a__="[SEP]"				,     a__="[PAD]"				,     a__="[CLS]"				,     a__="[MASK]"				,     a__=True				,     a__=None				,     **a__				,     ):
       super().__init__(
           a__				,     tokenizer_file=a__				,     do_lower_case=a__				,     unk_token=a__				,     sep_token=a__				,     pad_token=a__				,     cls_token=a__				,     mask_token=a__				,     tokenize_chinese_chars=a__				,     strip_accents=a__				,     **a__				,     )

       _lowerCAmelCase						:       int  			=     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get("""lowercase"""				,     a__  ) != do_lower_case
           or normalizer_state.get("""strip_accents"""				,     a__  ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""				,     a__  ) != tokenize_chinese_chars
       ):
         _lowerCAmelCase						:       Dict  			=     getattr(a__				,     normalizer_state.pop("""type"""  )  )
         _lowerCAmelCase						:       int  			=     do_lower_case
         _lowerCAmelCase						:       str  			=     strip_accents
         _lowerCAmelCase						:       Dict  			=     tokenize_chinese_chars
         _lowerCAmelCase						:       str  			=     normalizer_class(**a__  )

       _lowerCAmelCase						:       List[str]  			=     do_lower_case






     def     __A  (		self				,     a__				,     a__=None  ):
       _lowerCAmelCase						:       int  			=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       List[str]  			=     [self.sep_token_id]
       _lowerCAmelCase						:       Any  			=     [self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self._tokenizer.model.save(a__				,     name=a__  )
       return tuple(a__  )




