





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase				:       Optional[Any]	   =				get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class    lowerCAmelCase							(					__UpperCamelCase,  unittest.TestCase			):
 UpperCAmelCase__													=		ReformerTokenizer
 UpperCAmelCase__													=		ReformerTokenizerFast
 UpperCAmelCase__													=		True
 UpperCAmelCase__													=		False
 UpperCAmelCase__													=		True
 def      A_	(      self  :      Union[str, Any]	)			->	List[str]:
     super().setUp()

     lowerCamelCase__    :      int					 =		ReformerTokenizer(UpperCAmelCase			,     keep_accents=UpperCAmelCase	)
     tokenizer.save_pretrained(self.tmpdirname	)
 def      A_	(      self  :      Tuple	)			->	int:
     lowerCamelCase__    :      Tuple					 =		'<s>'
     lowerCamelCase__    :      Optional[Any]					 =		1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase	)			,     UpperCAmelCase	)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase	)			,     UpperCAmelCase	)
 def      A_	(      self  :      Optional[Any]	)			->	List[str]:
     lowerCamelCase__    :      Union[str, Any]					 =		list(self.get_tokenizer().get_vocab().keys()	)

     self.assertEqual(vocab_keys[0]			,     '<unk>'	)
     self.assertEqual(vocab_keys[1]			,     '<s>'	)
     self.assertEqual(vocab_keys[-1]			,     'j'	)
     self.assertEqual(len(UpperCAmelCase	)			,     1000	)
 def      A_	(      self  :      List[str]	)			->	Union[str, Any]:
     self.assertEqual(self.get_tokenizer().vocab_size			,     1000	)
 def      A_	(      self  :      Optional[int]	)			->	List[str]:
     if not self.test_rust_tokenizer:
         return

     lowerCamelCase__    :      List[Any]					 =		self.get_tokenizer()
     lowerCamelCase__    :      int					 =		self.get_rust_tokenizer()

     lowerCamelCase__    :      Optional[int]					 =		'I was born in 92000, and this is falsé.'

     lowerCamelCase__    :      Union[str, Any]					 =		tokenizer.tokenize(UpperCAmelCase	)
     lowerCamelCase__    :      str					 =		rust_tokenizer.tokenize(UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		tokenizer.encode(UpperCAmelCase			,     add_special_tokens=UpperCAmelCase	)
     lowerCamelCase__    :      int					 =		rust_tokenizer.encode(UpperCAmelCase			,     add_special_tokens=UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)

     lowerCamelCase__    :      Optional[Any]					 =		self.get_rust_tokenizer()
     lowerCamelCase__    :      Tuple					 =		tokenizer.encode(UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		rust_tokenizer.encode(UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)
 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      Optional[Any]=15	)			->	List[str]:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
             lowerCamelCase__    :      Tuple					 =		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase			,     **UpperCAmelCase	)

             # Simple input
             lowerCamelCase__    :      List[str]					 =		'This is a simple input'
             lowerCamelCase__    :      Tuple					 =		['This is a simple input 1', 'This is a simple input 2']
             lowerCamelCase__    :      Tuple					 =		('This is a simple input', 'This is a pair')
             lowerCamelCase__    :      int					 =		[
                 ('This is a simple input 1', 'This is a simple input 2'),
                 ('This is a simple pair 1', 'This is a simple pair 2'),
             ]

             # Simple input tests
             self.assertRaises(UpperCAmelCase			,     tokenizer_r.encode			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'	)

             # Simple input
             self.assertRaises(UpperCAmelCase			,     tokenizer_r.encode_plus			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'	)

             # Simple input
             self.assertRaises(
                 UpperCAmelCase			,     tokenizer_r.batch_encode_plus			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'			,     )

             # Pair input
             self.assertRaises(UpperCAmelCase			,     tokenizer_r.encode			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'	)

             # Pair input
             self.assertRaises(UpperCAmelCase			,     tokenizer_r.encode_plus			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'	)

             # Pair input
             self.assertRaises(
                 UpperCAmelCase			,     tokenizer_r.batch_encode_plus			,     UpperCAmelCase			,     max_length=UpperCAmelCase			,     padding='max_length'			,     )
 def      A_	(      self  :      Dict	)			->	Tuple:
     pass
 def      A_	(      self  :      List[str]	)			->	int:
     lowerCamelCase__    :      Any					 =		ReformerTokenizer(UpperCAmelCase			,     keep_accents=UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		tokenizer.tokenize('This is a test'	)
     self.assertListEqual(UpperCAmelCase			,     ['▁This', '▁is', '▁a', '▁t', 'est']	)

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [285, 46, 10, 170, 382]			,     )

     lowerCamelCase__    :      Tuple					 =		tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
     self.assertListEqual(
         UpperCAmelCase			,     [
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             'é',
             '.',
         ]			,     )
     lowerCamelCase__    :      List[str]					 =		tokenizer.convert_tokens_to_ids(UpperCAmelCase	)
     self.assertListEqual(
         UpperCAmelCase			,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			,     )

     lowerCamelCase__    :      Union[str, Any]					 =		tokenizer.convert_ids_to_tokens(UpperCAmelCase	)
     self.assertListEqual(
         UpperCAmelCase			,     [
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '<unk>',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             '<unk>',
             '.',
         ]			,     )
 @cached_property
 def      A_	(      self  :      str	)			->	List[str]:
     return ReformerTokenizer.from_pretrained('google/reformer-crime-and-punishment'	)
 @slow
 def      A_	(      self  :      List[Any]	)			->	Any:
     lowerCamelCase__    :      str					 =		'Hello World!'
     lowerCamelCase__    :      Any					 =		[126, 32, 262, 152, 38, 72, 287]

     self.assertListEqual(UpperCAmelCase			,     self.big_tokenizer.encode(UpperCAmelCase	)	)
 @slow
 def      A_	(      self  :      Any	)			->	List[str]:
     lowerCamelCase__    :      Optional[int]					 =		(
         'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
         ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
     )
     lowerCamelCase__    :      Dict					 =		[
         108,
         265,
         24,
         111,
         4,
         258,
         156,
         35,
         28,
         275,
         3,
         259,
         297,
         260,
         84,
         4,
         35,
         110,
         44,
         8,
         259,
         91,
         268,
         21,
         11,
         209,
         274,
         109,
         266,
         277,
         117,
         86,
         93,
         315,
         258,
         278,
         258,
         277,
         258,
         0,
         258,
         288,
         258,
         319,
         258,
         0,
         258,
         0,
         258,
         0,
         258,
         0,
         258,
         287,
         258,
         315,
         258,
         289,
         258,
         278,
         99,
         269,
         266,
         262,
         8,
         259,
         241,
         4,
         217,
         230,
         268,
         266,
         55,
         168,
         106,
         75,
         193,
         266,
         223,
         27,
         49,
         26,
         282,
         25,
         264,
         299,
         19,
         26,
         0,
         258,
         277,
         117,
         86,
         93,
         176,
         183,
         270,
         11,
         262,
         42,
         61,
         265,
     ]

     self.assertListEqual(UpperCAmelCase			,     self.big_tokenizer.encode(UpperCAmelCase	)	)
 @require_torch
 @slow
 def      A_	(      self  :      Dict	)			->	Optional[int]:
     import torch

     from transformers import ReformerConfig, ReformerModel

     # Build sequence
     lowerCamelCase__    :      Tuple					 =		list(self.big_tokenizer.get_vocab().keys()	)[:10]
     lowerCamelCase__    :      int					 =		' '.join(UpperCAmelCase	)
     lowerCamelCase__    :      Union[str, Any]					 =		self.big_tokenizer.encode_plus(UpperCAmelCase			,     return_tensors='pt'	)
     lowerCamelCase__    :      List[str]					 =		self.big_tokenizer.batch_encode_plus([sequence, sequence]			,     return_tensors='pt'	)

     lowerCamelCase__    :      Any					 =		ReformerConfig()
     # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
     lowerCamelCase__    :      int					 =		encoded_sequence['input_ids'].shape
     lowerCamelCase__    :      str					 =		ReformerModel(UpperCAmelCase	)

     # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
     assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

     with torch.no_grad():
         model(**UpperCAmelCase	)
         model(**UpperCAmelCase	)







 @slow
 def      A_	(      self  :      int	)			->	str:
     # fmt: off
     lowerCamelCase__    :      Union[str, Any]					 =		{'input_ids': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
     # fmt: on

     # This tokenizer does not know some characters like ")".
     # That is the reason why we use very simple texts here.
     # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
     lowerCamelCase__    :      Dict					 =		[
         'This is a very simple sentence.',
         'The quick brown fox jumps over the lazy dog.',
     ]

     self.tokenizer_integration_test_util(
         expected_encoding=UpperCAmelCase			,     model_name='google/reformer-crime-and-punishment'			,     revision='0e6c3decb8211d49bf881013425dc8b0448b3f5a'			,     padding=UpperCAmelCase			,     sequences=UpperCAmelCase			,     )












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase				:       Optional[int]	   =				logging.get_logger(__name__)

_UpperCAmelCase				:       Any	   =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase				:       int	   =				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase				:       int	   =				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase				:       Any	   =				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class    lowerCAmelCase							(					__UpperCamelCase			):
 UpperCAmelCase__													=		VOCAB_FILES_NAMES
 UpperCAmelCase__													=		PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__													=		PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__													=		RealmTokenizer
 def __init__(      self  :      Optional[int]			,     UpperCAmelCase  :      Tuple=None			,     UpperCAmelCase  :      Any=None			,     UpperCAmelCase  :      List[Any]=True			,     UpperCAmelCase  :      Optional[Any]="[UNK]"			,     UpperCAmelCase  :      Any="[SEP]"			,     UpperCAmelCase  :      Tuple="[PAD]"			,     UpperCAmelCase  :      List[Any]="[CLS]"			,     UpperCAmelCase  :      Union[str, Any]="[MASK]"			,     UpperCAmelCase  :      Optional[Any]=True			,     UpperCAmelCase  :      Any=None			,     **UpperCAmelCase  :      Optional[int]			,     )			->	str:
     super().__init__(
         UpperCAmelCase			,     tokenizer_file=UpperCAmelCase			,     do_lower_case=UpperCAmelCase			,     unk_token=UpperCAmelCase			,     sep_token=UpperCAmelCase			,     pad_token=UpperCAmelCase			,     cls_token=UpperCAmelCase			,     mask_token=UpperCAmelCase			,     tokenize_chinese_chars=UpperCAmelCase			,     strip_accents=UpperCAmelCase			,     **UpperCAmelCase			,     )

     lowerCamelCase__    :      List[Any]					 =		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
     if (
         normalizer_state.get('lowercase'			,     UpperCAmelCase	) != do_lower_case
         or normalizer_state.get('strip_accents'			,     UpperCAmelCase	) != strip_accents
         or normalizer_state.get('handle_chinese_chars'			,     UpperCAmelCase	) != tokenize_chinese_chars
     ):
         lowerCamelCase__    :      Optional[int]					 =		getattr(UpperCAmelCase			,     normalizer_state.pop('type'	)	)
         lowerCamelCase__    :      Optional[Any]					 =		do_lower_case
         lowerCamelCase__    :      str					 =		strip_accents
         lowerCamelCase__    :      Optional[Any]					 =		tokenize_chinese_chars
         lowerCamelCase__    :      int					 =		normalizer_class(**UpperCAmelCase	)

     lowerCamelCase__    :      str					 =		do_lower_case
 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      int			,     **UpperCAmelCase  :      int	)			->	List[Any]:
     lowerCamelCase__    :      List[Any]					 =		PaddingStrategy.MAX_LENGTH

     lowerCamelCase__    :      Optional[int]					 =		text
     lowerCamelCase__    :      Dict					 =		kwargs.pop('text_pair'			,     UpperCAmelCase	)
     lowerCamelCase__    :      List[Any]					 =		kwargs.pop('return_tensors'			,     UpperCAmelCase	)

     lowerCamelCase__    :      List[Any]					 =		{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase	):
         if batch_text_pair is not None:
             lowerCamelCase__    :      Tuple					 =		batch_text_pair[idx]
         else:
             lowerCamelCase__    :      Dict					 =		None

         lowerCamelCase__    :      Optional[int]					 =		super().__call__(UpperCAmelCase			,     UpperCAmelCase			,     return_tensors=UpperCAmelCase			,     **UpperCAmelCase	)

         lowerCamelCase__    :      Any					 =		encoded_candidates.get('input_ids'	)
         lowerCamelCase__    :      Union[str, Any]					 =		encoded_candidates.get('attention_mask'	)
         lowerCamelCase__    :      Tuple					 =		encoded_candidates.get('token_type_ids'	)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase	)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase	)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase	)

     lowerCamelCase__    :      int					 =		{key: item for key, item in output_data.items() if len(UpperCAmelCase	) != 0}

     return BatchEncoding(UpperCAmelCase			,     tensor_type=UpperCAmelCase	)
 def      A_	(      self  :      int			,     UpperCAmelCase  :      Optional[Any]			,     UpperCAmelCase  :      Optional[int]=None	)			->	List[str]:
     lowerCamelCase__    :      Tuple					 =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      List[int]			,     UpperCAmelCase  :      Optional[List[int]] = None	)			->	List[int]:
     lowerCamelCase__    :      List[Any]					 =		[self.sep_token_id]
     lowerCamelCase__    :      Optional[int]					 =		[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







 def      A_	(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[str] = None	)			->	Tuple[str]:
     lowerCamelCase__    :      int					 =		self._tokenizer.model.save(UpperCAmelCase			,     name=UpperCAmelCase	)
     return tuple(UpperCAmelCase	)





