


"""simple docstring"""







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast


class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[int]								=							0

       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
           lowerCAmelCase     :     str								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
           self.assertIsNotNone(lowerCAmelCase_						)
           self.assertIsInstance(lowerCAmelCase_		,     (BertTokenizer, BertTokenizerFast)						)
           self.assertGreater(len(lowerCAmelCase_						)		,     0						)

         for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
           lowerCAmelCase     :     List[str]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
           self.assertIsNotNone(lowerCAmelCase_						)
           self.assertIsInstance(lowerCAmelCase_		,     (GPTaTokenizer, GPTaTokenizerFast)						)
           self.assertGreater(len(lowerCAmelCase_						)		,     0						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
         self.assertIsInstance(lowerCAmelCase_		,     (BertTokenizer, BertTokenizerFast)						)
         self.assertEqual(tokenizer.vocab_size		,     12						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     int								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
         self.assertIsInstance(lowerCAmelCase_		,     (RobertaTokenizer, RobertaTokenizerFast)						)
         self.assertEqual(tokenizer.vocab_size		,     20						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[int]								=							AutoConfig.from_pretrained(lowerCAmelCase_						)
         self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)
         # Check that tokenizer_type ≠ model_type
         lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     config=lowerCAmelCase_						)
         self.assertIsInstance(lowerCAmelCase_		,     (BertTokenizer, BertTokenizerFast)						)
         self.assertEqual(tokenizer.vocab_size		,     12						)

       def        lowercase__							(						self						):






         """simple docstring"""





         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"		,     os.path.join(lowerCAmelCase_		,     "vocab.txt"						)						)

           lowerCAmelCase     :     Tuple								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     tokenizer_type="bert"		,     use_fast=lowerCAmelCase_						)
           self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"		,     os.path.join(lowerCAmelCase_		,     "vocab.json"						)						)
           shutil.copy("./tests/fixtures/merges.txt"		,     os.path.join(lowerCAmelCase_		,     "merges.txt"						)						)

           lowerCAmelCase     :     Any								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     tokenizer_type="gpt2"		,     use_fast=lowerCAmelCase_						)
           self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.txt"		,     os.path.join(lowerCAmelCase_		,     "vocab.txt"						)						)

           lowerCAmelCase     :     List[Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     tokenizer_type="bert"						)
           self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

         with tempfile.TemporaryDirectory() as tmp_dir:
           shutil.copy("./tests/fixtures/vocab.json"		,     os.path.join(lowerCAmelCase_		,     "vocab.json"						)						)
           shutil.copy("./tests/fixtures/merges.txt"		,     os.path.join(lowerCAmelCase_		,     "merges.txt"						)						)

           lowerCAmelCase     :     List[str]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     tokenizer_type="gpt2"						)
           self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

       def        lowercase__							(						self						):






         """simple docstring"""





         with pytest.raises(lowerCAmelCase_						):
           AutoTokenizer.from_pretrained("./"		,     tokenizer_type="xxx"						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           lowerCAmelCase     :     Optional[Any]								=							tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
           self.assertIsInstance(lowerCAmelCase_		,     (BertTokenizer, BertTokenizerFast)						)

           if isinstance(lowerCAmelCase_		,     lowerCAmelCase_						):
             self.assertEqual(tokenizer.basic_tokenizer.do_lower_case		,     lowerCAmelCase_						)
           else:
             self.assertEqual(tokenizer.do_lower_case		,     lowerCAmelCase_						)

           self.assertEqual(tokenizer.model_max_length		,     512						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
           with self.assertRaisesRegex(
               lowerCAmelCase_		,     "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"		,     ):
             lowerCAmelCase     :     Dict								=							tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[Any]								=							TOKENIZER_MAPPING.values()
         lowerCAmelCase     :     Any								=							[]

         for slow_tok, fast_tok in tokenizers:
           if slow_tok is not None:
             tokenizer_names.append(slow_tok.__name__						)

           if fast_tok is not None:
             tokenizer_names.append(fast_tok.__name__						)

         for tokenizer_name in tokenizer_names:
           # must find the right class
           tokenizer_class_from_name(lowerCAmelCase_						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"		,     use_fast=lowerCAmelCase_						)		,     lowerCAmelCase_						)
         self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)		,     lowerCAmelCase_						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							AutoTokenizer.from_pretrained("distilbert-base-uncased"		,     do_lower_case=lowerCAmelCase_						)
         lowerCAmelCase     :     List[Any]								=							"Hello, world. How are you?"
         lowerCAmelCase     :     int								=							tokenizer.tokenize(lowerCAmelCase_						)
         self.assertEqual("[UNK]"		,     tokens[0]						)

         lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained("microsoft/mpnet-base"		,     do_lower_case=lowerCAmelCase_						)
         lowerCAmelCase     :     str								=							tokenizer.tokenize(lowerCAmelCase_						)
         self.assertEqual("[UNK]"		,     tokens[0]						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
         self.assertEqual(type(lowerCAmelCase_						)		,     lowerCAmelCase_						)
         self.assertEqual(tokenizer.model_max_length		,     512						)
         self.assertEqual(tokenizer.vocab_size		,     30_000						)
         self.assertEqual(tokenizer.unk_token		,     "[UNK]"						)
         self.assertEqual(tokenizer.padding_side		,     "right"						)
         self.assertEqual(tokenizer.truncation_side		,     "right"						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
         self.assertIsInstance(lowerCAmelCase_		,     (BertTokenizer, BertTokenizerFast)						)
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase_						)
           lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)

         self.assertIsInstance(lowerCAmelCase_		,     tokenizer.__class__						)
         self.assertEqual(tokenizera.vocab_size		,     12						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained("ctrl"						)
         # There is no fast CTRL so this always gives us a slow tokenizer.
         self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[Any]								=							get_tokenizer_config("bert-base-cased"						)
         lowerCAmelCase     :     Optional[int]								=							config.pop("_commit_hash"		,     lowerCAmelCase_						)
         # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
         self.assertEqual(lowerCAmelCase_		,     {"do_lower_case": False}						)

         # This model does not have a tokenizer_config so we get back an empty dict.
         lowerCAmelCase     :     int								=							get_tokenizer_config(lowerCAmelCase_						)
         self.assertDictEqual(lowerCAmelCase_		,     {}						)

         # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
         lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase_						)
           lowerCAmelCase     :     Union[str, Any]								=							get_tokenizer_config(lowerCAmelCase_						)

         # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
         self.assertEqual(config["tokenizer_class"]		,     "BertTokenizer"						)

       def        lowercase__							(						self						):






         """simple docstring"""





         try:
           AutoConfig.register("custom"		,     lowerCAmelCase_						)

           AutoTokenizer.register(lowerCAmelCase_		,     slow_tokenizer_class=lowerCAmelCase_						)
           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(lowerCAmelCase_						):
             AutoTokenizer.register(lowerCAmelCase_		,     slow_tokenizer_class=lowerCAmelCase_						)

           lowerCAmelCase     :     Union[str, Any]								=							CustomTokenizer.from_pretrained(lowerCAmelCase_						)
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(lowerCAmelCase_						)

             lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
             self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         try:
           AutoConfig.register("custom"		,     lowerCAmelCase_						)

           # Can register in two steps
           AutoTokenizer.register(lowerCAmelCase_		,     slow_tokenizer_class=lowerCAmelCase_						)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,     (CustomTokenizer, None)						)
           AutoTokenizer.register(lowerCAmelCase_		,     fast_tokenizer_class=lowerCAmelCase_						)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,     (CustomTokenizer, CustomTokenizerFast)						)

           del TOKENIZER_MAPPING._extra_content[CustomConfig]
           # Can register in one step
           AutoTokenizer.register(
               lowerCAmelCase_		,     slow_tokenizer_class=lowerCAmelCase_		,     fast_tokenizer_class=lowerCAmelCase_						)
           self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,     (CustomTokenizer, CustomTokenizerFast)						)

           # Trying to register something existing in the Transformers library will raise an error
           with self.assertRaises(lowerCAmelCase_						):
             AutoTokenizer.register(lowerCAmelCase_		,     fast_tokenizer_class=lowerCAmelCase_						)

           # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
           # and that model does not have a tokenizer.json
           with tempfile.TemporaryDirectory() as tmp_dir:
             lowerCAmelCase     :     Dict								=							BertTokenizerFast.from_pretrained(lowerCAmelCase_						)
             bert_tokenizer.save_pretrained(lowerCAmelCase_						)
             lowerCAmelCase     :     Optional[Any]								=							CustomTokenizerFast.from_pretrained(lowerCAmelCase_						)

           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(lowerCAmelCase_						)

             lowerCAmelCase     :     Any								=							AutoTokenizer.from_pretrained(lowerCAmelCase_						)
             self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

             lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
             self.assertIsInstance(lowerCAmelCase_		,     lowerCAmelCase_						)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]

       def        lowercase__							(						self						):






         """simple docstring"""





         with self.assertRaises(lowerCAmelCase_						):
           lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
         # If remote code is disabled, we can't load this config.
         with self.assertRaises(lowerCAmelCase_						):
           lowerCAmelCase     :     Tuple								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_						)

         lowerCAmelCase     :     Dict								=							AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_						)
         self.assertTrue(tokenizer.special_attribute_present						)
         # Test tokenizer can be reloaded.
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase_						)
           lowerCAmelCase     :     Dict								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     trust_remote_code=lowerCAmelCase_						)
         self.assertTrue(reloaded_tokenizer.special_attribute_present						)

         if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizerFast"						)
           self.assertEqual(reloaded_tokenizer.__class__.__name__		,     "NewTokenizerFast"						)

           # Test we can also load the slow version
           lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
           self.assertTrue(tokenizer.special_attribute_present						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(lowerCAmelCase_						)
             lowerCAmelCase     :     int								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     trust_remote_code=lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
           self.assertEqual(reloaded_tokenizer.__class__.__name__		,     "NewTokenizer"						)
           self.assertTrue(reloaded_tokenizer.special_attribute_present						)
         else:
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
           self.assertEqual(reloaded_tokenizer.__class__.__name__		,     "NewTokenizer"						)

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         class       SCREAMING_SNAKE_CASE__				(		_lowerCamelCase				):
                """simple docstring"""


                a  :				Tuple   =False


         class       SCREAMING_SNAKE_CASE__				(		_lowerCamelCase				):
                """simple docstring"""


                a  :				List[str]   =NewTokenizer
                a  :				List[Any]   =False

         try:
           AutoConfig.register("custom"		,     lowerCAmelCase_						)
           AutoTokenizer.register(lowerCAmelCase_		,     slow_tokenizer_class=lowerCAmelCase_						)
           AutoTokenizer.register(lowerCAmelCase_		,     fast_tokenizer_class=lowerCAmelCase_						)
           # If remote code is not set, the default is to use local
           lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizerFast"						)
           self.assertFalse(tokenizer.special_attribute_present						)
           lowerCAmelCase     :     Tuple								=							AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,     use_fast=lowerCAmelCase_						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
           self.assertFalse(tokenizer.special_attribute_present						)

           # If remote code is disabled, we load the local one.
           lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizerFast"						)
           self.assertFalse(tokenizer.special_attribute_present						)
           lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
           self.assertFalse(tokenizer.special_attribute_present						)

           # If remote is enabled, we load from the Hub
           lowerCAmelCase     :     int								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizerFast"						)
           self.assertTrue(tokenizer.special_attribute_present						)
           lowerCAmelCase     :     Optional[int]								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer"		,     trust_remote_code=lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
           self.assertTrue(tokenizer.special_attribute_present						)

         finally:
           if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
           if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained(
             "hf-internal-testing/test_dynamic_tokenizer_legacy"		,     trust_remote_code=lowerCAmelCase_						)
         self.assertTrue(tokenizer.special_attribute_present						)
         if is_tokenizers_available():
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizerFast"						)

           # Test we can also load the slow version
           lowerCAmelCase     :     Tuple								=							AutoTokenizer.from_pretrained(
               "hf-internal-testing/test_dynamic_tokenizer_legacy"		,     trust_remote_code=lowerCAmelCase_		,     use_fast=lowerCAmelCase_						)
           self.assertTrue(tokenizer.special_attribute_present						)
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)
         else:
           self.assertEqual(tokenizer.__class__.__name__		,     "NewTokenizer"						)

       def        lowercase__							(						self						):






         """simple docstring"""





         with self.assertRaisesRegex(
             lowerCAmelCase_		,     "bert-base is not a local folder and is not a valid model identifier"						):
           lowerCAmelCase     :     int								=							AutoTokenizer.from_pretrained("bert-base"						)

       def        lowercase__							(						self						):






         """simple docstring"""





         with self.assertRaisesRegex(
             lowerCAmelCase_		,     r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
           lowerCAmelCase     :     Dict								=							AutoTokenizer.from_pretrained(lowerCAmelCase_		,     revision="aaaaaa"						)







       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
         with RequestCounter() as counter:
           lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
           self.assertEqual(counter.get_request_count		,     0						)
           self.assertEqual(counter.head_request_count		,     1						)
           self.assertEqual(counter.other_request_count		,     0						)







'''simple docstring'''







import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase    :							int      =					logging.get_logger(__name__)

lowercase    :							Union[str, Any]      =					{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}


class 						__UpperCAmelCase      (					_lowerCamelCase						):
							__lowercase      		=       """xlnet"""
							__lowercase      		=       ["""mems"""]
							__lowercase      		=       {
							    """n_token""": """vocab_size""",  # Backward compatibility
							    """hidden_size""": """d_model""",
							    """num_attention_heads""": """n_head""",
							    """num_hidden_layers""": """n_layer""",
							}


							def __init__(       self    ,   lowerCAmelCase_=3_20_00    ,   lowerCAmelCase_=10_24    ,   lowerCAmelCase_=24    ,   lowerCAmelCase_=16    ,   lowerCAmelCase_=40_96    ,   lowerCAmelCase_="gelu"    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_="bi"    ,   lowerCAmelCase_=0.02    ,   lowerCAmelCase_=1E-12    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=5_12    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=-1    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_="last"    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_="tanh"    ,   lowerCAmelCase_=0.1    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=5    ,   lowerCAmelCase_=1    ,   lowerCAmelCase_=2    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		vocab_size
								_snake_case   						=		d_model
								_snake_case   						=		n_layer
								_snake_case   						=		n_head
								if d_model % n_head != 0:
									raise ValueError(F'\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'  )
								if "d_head" in kwargs:
									if kwargs["d_head"] != d_model // n_head:
										raise ValueError(
										    F'`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})'  )
								_snake_case   						=		d_model // n_head
								_snake_case   						=		ff_activation
								_snake_case   						=		d_inner
								_snake_case   						=		untie_r
								_snake_case   						=		attn_type

								_snake_case   						=		initializer_range
								_snake_case   						=		layer_norm_eps

								_snake_case   						=		dropout
								_snake_case   						=		mem_len
								_snake_case   						=		reuse_len
								_snake_case   						=		bi_data
								_snake_case   						=		clamp_len
								_snake_case   						=		same_length

								_snake_case   						=		summary_type
								_snake_case   						=		summary_use_proj
								_snake_case   						=		summary_activation
								_snake_case   						=		summary_last_dropout
								_snake_case   						=		start_n_top
								_snake_case   						=		end_n_top

								_snake_case   						=		bos_token_id
								_snake_case   						=		pad_token_id
								_snake_case   						=		eos_token_id

								if "use_cache" in kwargs:
									warnings.warn(
									    'The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'
									    ' instead.'    ,   lowerCAmelCase_    ,   )
									_snake_case   						=		kwargs['use_cache']

								_snake_case   						=		use_mems_eval
								_snake_case   						=		use_mems_train
								super().__init__(pad_token_id=lowerCAmelCase_    ,   bos_token_id=lowerCAmelCase_    ,   eos_token_id=lowerCAmelCase_    ,   **lowerCAmelCase_  )


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								logger.info(F'The model {self.model_type} is one of the few models that has no sequence length limit.'  )
								return -1




							@max_position_embeddings.setter
							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								raise NotImplementedError(
								    F'The model {self.model_type} is one of the few models that has no sequence length limit.'  )

