import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402


__lowerCamelCase					:	Any       =  {
    '''return_dict''': False,
    '''output_hidden_states''': True,
    '''output_attentions''': True,
    '''torchscript''': True,
    '''torch_dtype''': '''float16''',
    '''use_bfloat16''': True,
    '''tf_legacy_loss''': True,
    '''pruned_heads''': {'''a''': 1},
    '''tie_word_embeddings''': False,
    '''is_decoder''': True,
    '''cross_attention_hidden_size''': 1_28,
    '''add_cross_attention''': True,
    '''tie_encoder_decoder''': True,
    '''max_length''': 50,
    '''min_length''': 3,
    '''do_sample''': True,
    '''early_stopping''': True,
    '''num_beams''': 3,
    '''num_beam_groups''': 3,
    '''diversity_penalty''': 0.5,
    '''temperature''': 2.0,
    '''top_k''': 10,
    '''top_p''': 0.7,
    '''typical_p''': 0.2,
    '''repetition_penalty''': 0.8,
    '''length_penalty''': 0.8,
    '''no_repeat_ngram_size''': 5,
    '''encoder_no_repeat_ngram_size''': 5,
    '''bad_words_ids''': [1, 2, 3],
    '''num_return_sequences''': 3,
    '''chunk_size_feed_forward''': 5,
    '''output_scores''': True,
    '''return_dict_in_generate''': True,
    '''forced_bos_token_id''': 2,
    '''forced_eos_token_id''': 3,
    '''remove_invalid_values''': True,
    '''architectures''': ['''BertModel'''],
    '''finetuning_task''': '''translation''',
    '''id2label''': {0: '''label'''},
    '''label2id''': {'''label''': '''0'''},
    '''tokenizer_class''': '''BertTokenizerFast''',
    '''prefix''': '''prefix''',
    '''bos_token_id''': 6,
    '''pad_token_id''': 7,
    '''eos_token_id''': 8,
    '''sep_token_id''': 9,
    '''decoder_start_token_id''': 10,
    '''exponential_decay_length_penalty''': (5, 1.01),
    '''suppress_tokens''': [0, 1],
    '''begin_suppress_tokens''': 2,
    '''task_specific_params''': {'''translation''': '''some_params'''},
    '''problem_type''': '''regression''',
}






@is_staging_test
class 		a__						(					unittest.TestCase					):

					@classmethod
					def 				__UpperCamelCase   (    cls      :       List[str]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    TOKEN
							HfFolder.save_token(_A    )

					@classmethod
					def 				__UpperCamelCase   (    cls      :       str    ):







							"""simple docstring"""






							try:
									delete_repo(token=cls._token,repo_id="test-config"    )
							except HTTPError:
									pass

							try:
									delete_repo(token=cls._token,repo_id="valid_org/test-config-org"    )
							except HTTPError:
									pass

							try:
									delete_repo(token=cls._token,repo_id="test-dynamic-config"    )
							except HTTPError:
									pass

					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Any				      =    BertConfig(
							    vocab_size=99,hidden_size=32,num_hidden_layers=5,num_attention_heads=4,intermediate_size=37    )
							config.push_to_hub("test-config",use_auth_token=self._token    )

							SCREAMING_SNAKE_CASE_  :		int				      =    BertConfig.from_pretrained(F'{USER}/test-config'    )
							for k, v in config.to_dict().items():
									if k != "transformers_version":
											self.assertEqual(_A,getattr(_A,_A    )    )

        # Reset repo
							delete_repo(token=self._token,repo_id="test-config"    )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
									config.save_pretrained(_A,repo_id="test-config",push_to_hub=_A,use_auth_token=self._token    )

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    BertConfig.from_pretrained(F'{USER}/test-config'    )
							for k, v in config.to_dict().items():
									if k != "transformers_version":
											self.assertEqual(_A,getattr(_A,_A    )    )

					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    BertConfig(
							    vocab_size=99,hidden_size=32,num_hidden_layers=5,num_attention_heads=4,intermediate_size=37    )
							config.push_to_hub("valid_org/test-config-org",use_auth_token=self._token    )

							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    BertConfig.from_pretrained("valid_org/test-config-org"    )
							for k, v in config.to_dict().items():
									if k != "transformers_version":
											self.assertEqual(_A,getattr(_A,_A    )    )

        # Reset repo
							delete_repo(token=self._token,repo_id="valid_org/test-config-org"    )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
									config.save_pretrained(
									    _A,repo_id="valid_org/test-config-org",push_to_hub=_A,use_auth_token=self._token    )

							SCREAMING_SNAKE_CASE_  :		int				      =    BertConfig.from_pretrained("valid_org/test-config-org"    )
							for k, v in config.to_dict().items():
									if k != "transformers_version":
											self.assertEqual(_A,getattr(_A,_A    )    )






					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							CustomConfig.register_for_auto_class()
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    CustomConfig(attribute=42    )

							config.push_to_hub("test-dynamic-config",use_auth_token=self._token    )

							# This has added the proper auto_map field to the config
							self.assertDictEqual(config.auto_map,{"AutoConfig": "custom_configuration.CustomConfig"}    )

							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    AutoConfig.from_pretrained(F'{USER}/test-dynamic-config',trust_remote_code=_A    )
							# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
							self.assertEqual(new_config.__class__.__name__,"CustomConfig"    )
							self.assertEqual(new_config.attribute,42    )







class 		a__						(					unittest.TestCase					):

					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		List[str]				      =    GPTaConfig()

							# attempt to modify each of int/float/bool/str config records and verify they were updated
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    c.n_embd + 1  # int
							SCREAMING_SNAKE_CASE_  :		Any				      =    c.resid_pdrop + 1.0  # float
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    not c.scale_attn_weights  # bool
							SCREAMING_SNAKE_CASE_  :		Any				      =    c.summary_type + "foo"  # str
							c.update_from_string(
							    F'n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}'    )
							self.assertEqual(_A,c.n_embd,"mismatch for key: n_embd"    )
							self.assertEqual(_A,c.resid_pdrop,"mismatch for key: resid_pdrop"    )
							self.assertEqual(_A,c.scale_attn_weights,"mismatch for key: scale_attn_weights"    )
							self.assertEqual(_A,c.summary_type,"mismatch for key: summary_type"    )

					def 				__UpperCamelCase   (    self      :       str    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    PretrainedConfig()
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    [key for key in base_config.__dict__ if key not in config_common_kwargs]
							# If this part of the test fails, you have arguments to addin config_common_kwargs above.
							self.assertListEqual(
							    _A,["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"]    )
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    [key for key, value in config_common_kwargs.items() if value == getattr(_A,_A    )]
							if len(_A    ) > 0:
									raise ValueError(
									    "The following keys are set with the default values in"
									    " `test_configuration_common.config_common_kwargs` pick another value for them:"
									    F' {", ".join(_A    )}.'    )

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							with self.assertRaises(_A    ):
									# config is in subfolder, the following should not work without specifying the subfolder
									SCREAMING_SNAKE_CASE_  :		Any				      =    BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder"    )

							SCREAMING_SNAKE_CASE_  :		Dict				      =    BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder",subfolder="bert"    )

							self.assertIsNotNone(_A    )

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    mock.Mock()
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    500
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    {}
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    HTTPError
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    {}

							# Download this model to make sure it's in the cache.
							SCREAMING_SNAKE_CASE_  :		str				      =    BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert"    )

							# Under the mock environment we get a 500 error when trying to reach the model.
							with mock.patch("requests.Session.request",return_value=_A    ) as mock_head:
									SCREAMING_SNAKE_CASE_  :		Dict				      =    BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert"    )
									# This check we did call the fake head request
									mock_head.assert_called()

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    BertConfig.from_pretrained(
							    "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"    )

					def 				__UpperCamelCase   (    self      :       Optional[Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Tuple				      =    AutoConfig.from_pretrained("bert-base-cased"    )
							SCREAMING_SNAKE_CASE_  :		int				      =    ["config.4.0.0.json"]

							with tempfile.TemporaryDirectory() as tmp_dir:
									configuration.save_pretrained(_A    )
									SCREAMING_SNAKE_CASE_  :		str				      =    2
									json.dump(configuration.to_dict(),open(os.path.join(_A,"config.4.0.0.json"    ),"w"    )    )

									# This should pick the new configuration file as the version of Transformers is > 4.0.0
									SCREAMING_SNAKE_CASE_  :		List[Any]				      =    AutoConfig.from_pretrained(_A    )
									self.assertEqual(new_configuration.hidden_size,2    )

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old configuration file as the version of Transformers is < 4.42.0
									SCREAMING_SNAKE_CASE_  :		str				      =    ["config.42.0.0.json"]
									SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    768
									configuration.save_pretrained(_A    )
									shutil.move(os.path.join(_A,"config.4.0.0.json"    ),os.path.join(_A,"config.42.0.0.json"    )    )
									SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    AutoConfig.from_pretrained(_A    )
									self.assertEqual(new_configuration.hidden_size,768    )






					def 				__UpperCamelCase   (    self      :       str    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    "hf-internal-testing/test-two-configs"

							import transformers as new_transformers

							SCREAMING_SNAKE_CASE_  :		Dict				      =    "v4.0.0"
							SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_  :		List[str]				      =    new_transformers.models.auto.AutoConfig.from_pretrained(
							    _A,return_unused_kwargs=_A    )
							self.assertEqual(new_configuration.hidden_size,2    )
							# This checks `_configuration_file` ia not kept in the kwargs by mistake.
							self.assertDictEqual(_A,{}    )

							# Testing an older version by monkey-patching the version in the module it's used.
							import transformers as old_transformers

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    "v3.0.0"
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    old_transformers.models.auto.AutoConfig.from_pretrained(_A    )
							self.assertEqual(old_configuration.hidden_size,768    )

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__lowerCamelCase					:	Optional[Any]       =  logging.get_logger(__name__)

__lowerCamelCase					:	Dict       =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase					:	int       =  {
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

__lowerCamelCase					:	Any       =  {
    '''junnyu/roformer_chinese_small''': 15_36,
    '''junnyu/roformer_chinese_base''': 15_36,
    '''junnyu/roformer_chinese_char_small''': 5_12,
    '''junnyu/roformer_chinese_char_base''': 5_12,
    '''junnyu/roformer_small_discriminator''': 1_28,
    '''junnyu/roformer_small_generator''': 1_28,
}


__lowerCamelCase					:	Union[str, Any]       =  {
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}






class 		a__						(					A__					):
					A	       =							VOCAB_FILES_NAMES
					A	       =							PRETRAINED_VOCAB_FILES_MAP
					A	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A	       =							PRETRAINED_INIT_CONFIGURATION
					A	       =							RoFormerTokenizer

					def __init__(    self      :       List[str],_A      :       int=None,_A      :       int=None,_A      :       int=True,_A      :       List[Any]="[UNK]",_A      :       Tuple="[SEP]",_A      :       List[Any]="[PAD]",_A      :       Optional[int]="[CLS]",_A      :       Optional[Any]="[MASK]",_A      :       Optional[int]=True,_A      :       List[str]=None,**_A      :       List[Any],):







							"""simple docstring"""






							super().__init__(
							    _A,tokenizer_file=_A,do_lower_case=_A,unk_token=_A,sep_token=_A,pad_token=_A,cls_token=_A,mask_token=_A,tokenize_chinese_chars=_A,strip_accents=_A,**_A,)

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
							if (
							    pre_tok_state.get("lowercase",_A    ) != do_lower_case
							    or pre_tok_state.get("strip_accents",_A    ) != strip_accents
							):
									SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    getattr(_A,pre_tok_state.pop("type"    )    )
									SCREAMING_SNAKE_CASE_  :		Any				      =    do_lower_case
									SCREAMING_SNAKE_CASE_  :		List[str]				      =    strip_accents
									SCREAMING_SNAKE_CASE_  :		str				      =    pre_tok_class(**_A    )

							SCREAMING_SNAKE_CASE_  :		List[str]				      =    do_lower_case

					def __getstate__(    self      :       Optional[int]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    self.__dict__.copy()
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    BertPreTokenizer()
							return state

					def __setstate__(    self      :       List[Any],_A      :       Union[str, Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    d
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    self.__dict__["_tokenizer"].get_vocab()
							SCREAMING_SNAKE_CASE_  :		Any				      =    PreTokenizer.custom(JiebaPreTokenizer(_A    )    )

					def 				__UpperCamelCase   (    self      :       Union[str, Any],_A      :       List[Any],_A      :       str=None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output

					def 				__UpperCamelCase   (    self      :       str,_A      :       List[int],_A      :       Optional[List[int]] = None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    [self.sep_token_id]
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    [self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep    ) * [0]
							return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

					def 				__UpperCamelCase   (    self      :       int,_A      :       str,_A      :       Optional[str] = None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    self._tokenizer.model.save(_A,name=_A    )
							return tuple(_A    )






					def 				__UpperCamelCase   (    self      :       int,_A      :       Optional[int],_A      :       List[Any]=None,_A      :       Tuple=None,_A      :       str=False,**_A      :       List[Any],):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    BertPreTokenizer()
							return super().save_pretrained(_A,_A,_A,_A,**_A    )
