'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_:     int  =get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class        __A      ( UpperCamelCase__		,	unittest.TestCase	):
    a__       :						Any          =  XLNetTokenizer
    a__       :						Tuple          =  XLNetTokenizerFast
    a__       :						int          =  True
    a__       :						Optional[int]          =  True



    def 					_lowercase						(self    :    Union[str, Any] ):
         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase_           =   XLNetTokenizer(__a   ,			keep_accents=__a )
         tokenizer.sanitize_special_tokens()
         tokenizer.save_pretrained(self.tmpdirname )



    def 					_lowercase						(self    :    Dict ):
         UpperCAmelCase_           =   "<s>"
         UpperCAmelCase_           =   1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a )   ,			__a )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a )   ,			__a )



    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   list(self.get_tokenizer().get_vocab().keys() )

         self.assertEqual(vocab_keys[0]   ,			"<unk>" )
         self.assertEqual(vocab_keys[1]   ,			"<s>" )
         self.assertEqual(vocab_keys[-1]   ,			"<eod>" )
         self.assertEqual(len(__a )   ,			1006 )



    def 					_lowercase						(self    :    Tuple ):
         self.assertEqual(self.get_tokenizer().vocab_size   ,			1000 )



    def 					_lowercase						(self    :    Tuple ):
         UpperCAmelCase_           =   XLNetTokenizer(__a   ,			keep_accents=__a )

         UpperCAmelCase_           =   tokenizer.tokenize("This is a test" )
         self.assertListEqual(__a   ,			["▁This", "▁is", "▁a", "▁t", "est"] )

         self.assertListEqual(tokenizer.convert_tokens_to_ids(__a )   ,			[285, 46, 10, 170, 382] )

         UpperCAmelCase_           =   tokenizer.tokenize("I was born in 92000, and this is falsé." )
         self.assertListEqual(
             __a   ,			[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]   ,			)
         UpperCAmelCase_           =   tokenizer.convert_tokens_to_ids(__a )
         self.assertListEqual(__a   ,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

         UpperCAmelCase_           =   tokenizer.convert_ids_to_tokens(__a )
         self.assertListEqual(
             __a   ,			[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]   ,			)



    def 					_lowercase						(self    :    Any ):
         UpperCAmelCase_           =   XLNetTokenizer(__a   ,			do_lower_case=__a )
         UpperCAmelCase_           =   tokenizer.tokenize("I was born in 92000, and this is falsé." )
         self.assertListEqual(
             __a   ,			[
                 SPIECE_UNDERLINE + "",
                 "i",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "se",
                 ".",
             ]   ,			)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )   ,			["▁he", "ll", "o"] )



    def 					_lowercase						(self    :    int ):
         UpperCAmelCase_           =   XLNetTokenizer(__a   ,			do_lower_case=__a )
         UpperCAmelCase_           =   tokenizer.tokenize("I was born in 92000, and this is falsé." )
         self.assertListEqual(
             __a   ,			[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "se",
                 ".",
             ]   ,			)



    @slow
    def 					_lowercase						(self    :    str ):
         UpperCAmelCase_           =   XLNetTokenizer.from_pretrained("xlnet-base-cased" )

         UpperCAmelCase_           =   tokenizer.encode("sequence builders"   ,			add_special_tokens=__a )
         UpperCAmelCase_           =   tokenizer.encode("multi-sequence build"   ,			add_special_tokens=__a )

         UpperCAmelCase_           =   tokenizer.build_inputs_with_special_tokens(__a )
         UpperCAmelCase_           =   tokenizer.build_inputs_with_special_tokens(__a   ,			__a )

         assert encoded_sentence == text + [4, 3]
         assert encoded_pair == text + [4] + text_a + [4, 3]






    @slow
    def 					_lowercase						(self    :    Optional[int] ):
         # fmt: off
         UpperCAmelCase_           =   {"input_ids": [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__a   ,			model_name="xlnet-base-cased"   ,			revision="c841166438c31ec7ca9a106dee7bb312b73ae511"   ,			)





'''simple docstring'''





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer




class        __A      ( UpperCamelCase__	):



    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   tempfile.mkdtemp()
         UpperCAmelCase_           =   5

         # Realm tok
         UpperCAmelCase_           =   [
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "test",
             "question",
             "this",
             "is",
             "the",
             "first",
             "second",
             "third",
             "fourth",
             "fifth",
             "record",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         UpperCAmelCase_           =   os.path.join(self.tmpdirname   ,			"realm_tokenizer" )
         os.makedirs(__a   ,			exist_ok=__a )
         UpperCAmelCase_           =   os.path.join(__a   ,			VOCAB_FILES_NAMES["vocab_file"] )
         with open(self.vocab_file   ,			"w"   ,			encoding="utf-8" ) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )

         UpperCAmelCase_           =   os.path.join(self.tmpdirname   ,			"realm_block_records" )
         os.makedirs(__a   ,			exist_ok=__a )



    def 					_lowercase						(self    :    Optional[Any] ):
         return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,			"realm_tokenizer" ) )



    def 					_lowercase						(self    :    Any ):
         shutil.rmtree(self.tmpdirname )



    def 					_lowercase						(self    :    List[Any] ):
         UpperCAmelCase_           =   RealmConfig(num_block_records=self.num_block_records )
         return config



    def 					_lowercase						(self    :    List[str] ):
         UpperCAmelCase_           =   Dataset.from_dict(
             {
                 "id": ["0", "1"],
                 "question": ["foo", "bar"],
                 "answers": [["Foo", "Bar"], ["Bar"]],
             } )
         return dataset



    def 					_lowercase						(self    :    Any ):
         UpperCAmelCase_           =   np.array(
             [
                 B"This is the first record",
                 B"This is the second record",
                 B"This is the third record",
                 B"This is the fourth record",
                 B"This is the fifth record",
                 B"This is a longer longer longer record",
             ]   ,			dtype=__a   ,			)
         return block_records



    def 					_lowercase						(self    :    Union[str, Any] ):
         UpperCAmelCase_           =   RealmRetriever(
             block_records=self.get_dummy_block_records()   ,			tokenizer=self.get_tokenizer()   ,			)
         return retriever



    def 					_lowercase						(self    :    int ):
         UpperCAmelCase_           =   self.get_config()
         UpperCAmelCase_           =   self.get_dummy_retriever()
         UpperCAmelCase_           =   retriever.tokenizer

         UpperCAmelCase_           =   np.array([0, 3]   ,			dtype="long" )
         UpperCAmelCase_           =   tokenizer(["Test question"] ).input_ids
         UpperCAmelCase_           =   tokenizer(
             ["the fourth"]   ,			add_special_tokens=__a   ,			return_token_type_ids=__a   ,			return_attention_mask=__a   ,			).input_ids
         UpperCAmelCase_           =   config.reader_seq_len

         UpperCAmelCase_    , UpperCAmelCase_    , UpperCAmelCase_    , UpperCAmelCase_           =   retriever(
             __a   ,			__a   ,			answer_ids=__a   ,			max_length=__a   ,			return_tensors="np" )

         self.assertEqual(len(__a )   ,			2 )
         self.assertEqual(len(__a )   ,			2 )
         self.assertEqual(len(__a )   ,			2 )
         self.assertEqual(concat_inputs.input_ids.shape   ,			(2, 10) )
         self.assertEqual(concat_inputs.attention_mask.shape   ,			(2, 10) )
         self.assertEqual(concat_inputs.token_type_ids.shape   ,			(2, 10) )
         self.assertEqual(concat_inputs.special_tokens_mask.shape   ,			(2, 10) )
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0] )   ,			["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]   ,			)
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1] )   ,			["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]   ,			)



    def 					_lowercase						(self    :    List[Any] ):
         UpperCAmelCase_           =   self.get_config()
         UpperCAmelCase_           =   self.get_dummy_retriever()
         UpperCAmelCase_           =   retriever.tokenizer

         UpperCAmelCase_           =   np.array([0, 3, 5]   ,			dtype="long" )
         UpperCAmelCase_           =   tokenizer(["Test question"] ).input_ids
         UpperCAmelCase_           =   tokenizer(
             ["the fourth", "longer longer"]   ,			add_special_tokens=__a   ,			return_token_type_ids=__a   ,			return_attention_mask=__a   ,			).input_ids
         UpperCAmelCase_           =   config.reader_seq_len

         UpperCAmelCase_    , UpperCAmelCase_    , UpperCAmelCase_    , UpperCAmelCase_           =   retriever(
             __a   ,			__a   ,			answer_ids=__a   ,			max_length=__a   ,			return_tensors="np" )

         self.assertEqual([False, True, True]   ,			__a )
         self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]   ,			__a )
         self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]   ,			__a )






    def 					_lowercase						(self    :    Optional[Any] ):
         UpperCAmelCase_           =   self.get_dummy_retriever()
         retriever.save_pretrained(os.path.join(self.tmpdirname   ,			"realm_block_records" ) )

         # Test local path
         UpperCAmelCase_           =   retriever.from_pretrained(os.path.join(self.tmpdirname   ,			"realm_block_records" ) )
         self.assertEqual(retriever.block_records[0]   ,			B"This is the first record" )

         # Test mocked remote path
         with patch("transformers.models.realm.retrieval_realm.hf_hub_download" ) as mock_hf_hub_download:
              UpperCAmelCase_           =   os.path.join(
                  os.path.join(self.tmpdirname   ,			"realm_block_records" )   ,			_REALM_BLOCK_RECORDS_FILENAME )
              UpperCAmelCase_           =   RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa" )

         self.assertEqual(retriever.block_records[0]   ,			B"This is the first record" )




