
import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


A     :	Union[str, Any]    = {
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 1_2_8,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 5_0,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 1_0,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 1_0,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}







@is_staging_test
class 			__A( unittest.TestCase			):




				@classmethod
				def 				SCREAMING_SNAKE_CASE_		(     cls				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    TOKEN
								HfFolder.save_token(_snake_case				)




				@classmethod
				def 				SCREAMING_SNAKE_CASE_		(     cls				)    ->      Any:


								'''simple docstring'''




								try:
												delete_repo(token=cls._token		,     repo_id='''test-config'''				)
								except HTTPError:
												pass

								try:
												delete_repo(token=cls._token		,     repo_id='''valid_org/test-config-org'''				)
								except HTTPError:
												pass

								try:
												delete_repo(token=cls._token		,     repo_id='''test-dynamic-config'''				)
								except HTTPError:
												pass




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      str:


								'''simple docstring'''




								__a										=    BertConfig(
								    vocab_size=99		,     hidden_size=32		,     num_hidden_layers=5		,     num_attention_heads=4		,     intermediate_size=37				)
								config.push_to_hub('''test-config'''		,     use_auth_token=self._token				)

								__a										=    BertConfig.from_pretrained(F"""{USER}/test-config"""				)
								for k, v in config.to_dict().items():
												if k != "transformers_version":
																self.assertEqual(_snake_case		,     getattr(_snake_case		,     _snake_case				)				)

        # Reset repo
								delete_repo(token=self._token		,     repo_id='''test-config'''				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												config.save_pretrained(_snake_case		,     repo_id='''test-config'''		,     push_to_hub=_snake_case		,     use_auth_token=self._token				)

								__a										=    BertConfig.from_pretrained(F"""{USER}/test-config"""				)
								for k, v in config.to_dict().items():
												if k != "transformers_version":
																self.assertEqual(_snake_case		,     getattr(_snake_case		,     _snake_case				)				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      List[Any]:


								'''simple docstring'''




								__a										=    BertConfig(
								    vocab_size=99		,     hidden_size=32		,     num_hidden_layers=5		,     num_attention_heads=4		,     intermediate_size=37				)
								config.push_to_hub('''valid_org/test-config-org'''		,     use_auth_token=self._token				)

								__a										=    BertConfig.from_pretrained('''valid_org/test-config-org'''				)
								for k, v in config.to_dict().items():
												if k != "transformers_version":
																self.assertEqual(_snake_case		,     getattr(_snake_case		,     _snake_case				)				)

        # Reset repo
								delete_repo(token=self._token		,     repo_id='''valid_org/test-config-org'''				)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												config.save_pretrained(
												    _snake_case		,     repo_id='''valid_org/test-config-org'''		,     push_to_hub=_snake_case		,     use_auth_token=self._token				)

								__a										=    BertConfig.from_pretrained('''valid_org/test-config-org'''				)
								for k, v in config.to_dict().items():
												if k != "transformers_version":
																self.assertEqual(_snake_case		,     getattr(_snake_case		,     _snake_case				)				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      List[Any]:


								'''simple docstring'''




								CustomConfig.register_for_auto_class()
								__a										=    CustomConfig(attribute=42				)

								config.push_to_hub('''test-dynamic-config'''		,     use_auth_token=self._token				)

								# This has added the proper auto_map field to the config
								self.assertDictEqual(config.auto_map		,     {'''AutoConfig''': '''custom_configuration.CustomConfig'''}				)

								__a										=    AutoConfig.from_pretrained(F"""{USER}/test-dynamic-config"""		,     trust_remote_code=_snake_case				)
								# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
								self.assertEqual(new_config.__class__.__name__		,     '''CustomConfig'''				)
								self.assertEqual(new_config.attribute		,     42				)







class 			__A( unittest.TestCase			):




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    GPTaConfig()

								# attempt to modify each of int/float/bool/str config records and verify they were updated
								__a										=    c.n_embd + 1  # int
								__a										=    c.resid_pdrop + 1.0  # float
								__a										=    not c.scale_attn_weights  # bool
								__a										=    c.summary_type + '''foo'''  # str
								c.update_from_string(
								    F"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""				)
								self.assertEqual(_snake_case		,     c.n_embd		,     '''mismatch for key: n_embd'''				)
								self.assertEqual(_snake_case		,     c.resid_pdrop		,     '''mismatch for key: resid_pdrop'''				)
								self.assertEqual(_snake_case		,     c.scale_attn_weights		,     '''mismatch for key: scale_attn_weights'''				)
								self.assertEqual(_snake_case		,     c.summary_type		,     '''mismatch for key: summary_type'''				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Any:


								'''simple docstring'''




								__a										=    PretrainedConfig()
								__a										=    [key for key in base_config.__dict__ if key not in config_common_kwargs]
								# If this part of the test fails, you have arguments to addin config_common_kwargs above.
								self.assertListEqual(
								    _snake_case		,     ['''is_encoder_decoder''', '''_name_or_path''', '''_commit_hash''', '''transformers_version''']				)
								__a										=    [key for key, value in config_common_kwargs.items() if value == getattr(_snake_case		,     _snake_case				)]
								if len(_snake_case				) > 0:
												raise ValueError(
												    '''The following keys are set with the default values in'''
												    ''' `test_configuration_common.config_common_kwargs` pick another value for them:'''
												    F""" {', '.join(_snake_case				)}."""				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      List[str]:


								'''simple docstring'''




								with self.assertRaises(_snake_case				):
												# config is in subfolder, the following should not work without specifying the subfolder
												__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert-subfolder'''				)

								__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert-subfolder'''		,     subfolder='''bert'''				)

								self.assertIsNotNone(_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      str:


								'''simple docstring'''




								__a										=    mock.Mock()
								__a										=    500
								__a										=    {}
								__a										=    HTTPError
								__a										=    {}

								# Download this model to make sure it's in the cache.
								__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)

								# Under the mock environment we get a 500 error when trying to reach the model.
								with mock.patch('''requests.Session.request'''		,     return_value=_snake_case				) as mock_head:
												__a										=    BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
												# This check we did call the fake head request
												mock_head.assert_called()




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      Any:


								'''simple docstring'''




								__a										=    BertConfig.from_pretrained(
								    '''https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json'''				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      str:


								'''simple docstring'''




								__a										=    AutoConfig.from_pretrained('''bert-base-cased'''				)
								__a										=    ['''config.4.0.0.json''']

								with tempfile.TemporaryDirectory() as tmp_dir:
												configuration.save_pretrained(_snake_case				)
												__a										=    2
												json.dump(configuration.to_dict()		,     open(os.path.join(_snake_case		,     '''config.4.0.0.json'''				)		,     '''w'''				)				)

												# This should pick the new configuration file as the version of Transformers is > 4.0.0
												__a										=    AutoConfig.from_pretrained(_snake_case				)
												self.assertEqual(new_configuration.hidden_size		,     2				)

												# Will need to be adjusted if we reach v42 and this test is still here.
												# Should pick the old configuration file as the version of Transformers is < 4.42.0
												__a										=    ['''config.42.0.0.json''']
												__a										=    768
												configuration.save_pretrained(_snake_case				)
												shutil.move(os.path.join(_snake_case		,     '''config.4.0.0.json'''				)		,     os.path.join(_snake_case		,     '''config.42.0.0.json'''				)				)
												__a										=    AutoConfig.from_pretrained(_snake_case				)
												self.assertEqual(new_configuration.hidden_size		,     768				)




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      str:


								'''simple docstring'''




								__a										=    '''hf-internal-testing/test-two-configs'''

								import transformers as new_transformers

								__a										=    '''v4.0.0'''
								__a							,     __a										=    new_transformers.models.auto.AutoConfig.from_pretrained(
								    _snake_case		,     return_unused_kwargs=_snake_case				)
								self.assertEqual(new_configuration.hidden_size		,     2				)
								# This checks `_configuration_file` ia not kept in the kwargs by mistake.
								self.assertDictEqual(_snake_case		,     {}				)

								# Testing an older version by monkey-patching the version in the module it's used.
								import transformers as old_transformers

								__a										=    '''v3.0.0'''
								__a										=    old_transformers.models.auto.AutoConfig.from_pretrained(_snake_case				)
								self.assertEqual(old_configuration.hidden_size		,     768				)

import os
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Dict, List, Optional, Union

import torch
from filelock import FileLock
from torch.utils.data import Dataset

from ...models.auto.modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from ..processors.squad import SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features


__A				=logging.get_logger(__name__)

__A				=list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
__A				=tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)




@dataclass
class     _SCREAMING_SNAKE_CASE     :
	lowerCAmelCase__         = field(
	    default=snake_case_			,       metadata={'help': 'Model type selected in the list: ' + ', '.join(snake_case_						)}						)
	lowerCAmelCase__         = field(
	    default=snake_case_			,       metadata={'help': 'The input data dir. Should contain the .json files for the SQuAD task.'}						)
	lowerCAmelCase__         = field(
	    default=1_28			,       metadata={
	        'help': (
	            'The maximum total input sequence length after tokenization. Sequences longer '
	            'than this will be truncated, sequences shorter will be padded.'
	        )
	    }			,       )
	lowerCAmelCase__         = field(
	    default=1_28			,       metadata={'help': 'When splitting up a long document into chunks, how much stride to take between chunks.'}			,       )
	lowerCAmelCase__         = field(
	    default=64			,       metadata={
	        'help': (
	            'The maximum number of tokens for the question. Questions longer than this will '
	            'be truncated to this length.'
	        )
	    }			,       )
	lowerCAmelCase__         = field(
	    default=30			,       metadata={
	        'help': (
	            'The maximum length of an answer that can be generated. This is needed because the start '
	            'and end predictions are not conditioned on one another.'
	        )
	    }			,       )
	lowerCAmelCase__         = field(
	    default=snake_case_			,       metadata={'help': 'Overwrite the cached training and evaluation sets'}						)
	lowerCAmelCase__         = field(
	    default=snake_case_			,       metadata={'help': 'If true, the SQuAD examples contain some that do not have an answer.'}						)
	lowerCAmelCase__         = field(
	    default=0.0			,       metadata={'help': 'If null_score - best_non_null is greater than the threshold predict null.'}						)
	lowerCAmelCase__         = field(
	    default=20			,       metadata={'help': 'If null_score - best_non_null is greater than the threshold predict null.'}						)
	lowerCAmelCase__         = field(
	    default=0			,       metadata={
	        'help': (
	            'language id of input for language-specific xlm models (see'
	            ' tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)'
	        )
	    }			,       )
	lowerCAmelCase__         = field(default=1			,       metadata={'help': 'multiple threads for converting example to features'}						)




class     _SCREAMING_SNAKE_CASE     (      snake_case_						):
	lowerCAmelCase__         = 'train'
	lowerCAmelCase__         = 'dev'




class     _SCREAMING_SNAKE_CASE     (      snake_case_						):
	lowerCAmelCase__         = 42
	lowerCAmelCase__         = 42
	lowerCAmelCase__         = 42
	lowerCAmelCase__         = 42



	def __init__(    self						,  lowercase						,  lowercase						,  lowercase = None						,  lowercase = Split.train						,  lowercase = False						,  lowercase = None						,  lowercase = "pt"						,  )				->       List[str]:
					lowerCamelCase_						=			args
					lowerCamelCase_						=			is_language_sensitive
					lowerCamelCase_						=			SquadVaProcessor() if args.version_2_with_negative else SquadVaProcessor()
					if isinstance(lowercase						,  lowercase ):
									try:
													lowerCamelCase_						=			Split[mode]
									except KeyError:
													raise KeyError("mode is not a valid split name" )
					lowerCamelCase_						=			mode
					# Load data features from cache or dataset file
					lowerCamelCase_						=			"v2" if args.version_2_with_negative else "v1"
					lowerCamelCase_						=			os.path.join(
					    cache_dir if cache_dir is not None else args.data_dir						,  f'cached_{mode.value}_{tokenizer.__class__.__name__}_{args.max_seq_length}_{version_tag}'						,  )

					# Make sure only the first process in distributed training processes the dataset,
					# and the others will use the cache.
					lowerCamelCase_						=			cached_features_file + ".lock"
					with FileLock(lowercase ):
									if os.path.exists(lowercase ) and not args.overwrite_cache:
													lowerCamelCase_						=			time.time()
													lowerCamelCase_						=			torch.load(lowercase )

													# Legacy cache files have only features, while new cache files
													# will have dataset and examples also.
													lowerCamelCase_						=			self.old_features["features"]
													lowerCamelCase_						=			self.old_features.get("dataset"						,  lowercase )
													lowerCamelCase_						=			self.old_features.get("examples"						,  lowercase )
													logger.info(
													    f'Loading features from cached file {cached_features_file} [took %.3f s]'						,  time.time() - start )

													if self.dataset is None or self.examples is None:
																	logger.warning(
																	    f'Deleting cached file {cached_features_file} will allow dataset and examples to be cached in'
																	    " future run" )
									else:
													if mode == Split.dev:
																	lowerCamelCase_						=			self.processor.get_dev_examples(args.data_dir )
													else:
																	lowerCamelCase_						=			self.processor.get_train_examples(args.data_dir )

													lowerCamelCase_	,    lowerCamelCase_						=			squad_convert_examples_to_features(
													    examples=self.examples						,  tokenizer=lowercase						,  max_seq_length=args.max_seq_length						,  doc_stride=args.doc_stride						,  max_query_length=args.max_query_length						,  is_training=mode == Split.train						,  threads=args.threads						,  return_dataset=lowercase						,  )

													lowerCamelCase_						=			time.time()
													torch.save(
													    {"features": self.features, "dataset": self.dataset, "examples": self.examples}						,  lowercase						,  )
													# ^ This seems to take a lot of time so I want to investigate why and how we can improve.
													logger.info(
													    f'Saving features into cached file {cached_features_file} [took {time.time() - start:.3f} s]' )



	def __len__(    self )				->       Tuple:
					return len(self.features )



	def __getitem__(    self						,  lowercase )				->       Dict[str, torch.Tensor]:
					# Convert to Tensors and build dataset
					lowerCamelCase_						=			self.features[i]

					lowerCamelCase_						=			torch.tensor(feature.input_ids						,  dtype=torch.long )
					lowerCamelCase_						=			torch.tensor(feature.attention_mask						,  dtype=torch.long )
					lowerCamelCase_						=			torch.tensor(feature.token_type_ids						,  dtype=torch.long )
					lowerCamelCase_						=			torch.tensor(feature.cls_index						,  dtype=torch.long )
					lowerCamelCase_						=			torch.tensor(feature.p_mask						,  dtype=torch.float )
					lowerCamelCase_						=			torch.tensor(feature.is_impossible						,  dtype=torch.float )

					lowerCamelCase_						=			{
					    "input_ids": input_ids,
					    "attention_mask": attention_mask,
					    "token_type_ids": token_type_ids,
					}

					if self.args.model_type in ["xlm", "roberta", "distilbert", "camembert"]:
									del inputs["token_type_ids"]

					if self.args.model_type in ["xlnet", "xlm"]:
									inputs.update({"cls_index": cls_index, "p_mask": p_mask} )
									if self.args.version_2_with_negative:
													inputs.update({"is_impossible": is_impossible} )
									if self.is_language_sensitive:
													inputs.update({"langs": (torch.ones(input_ids.shape						,  dtype=torch.intaa ) * self.args.lang_id)} )

					if self.mode == Split.train:
									lowerCamelCase_						=			torch.tensor(feature.start_position						,  dtype=torch.long )
									lowerCamelCase_						=			torch.tensor(feature.end_position						,  dtype=torch.long )
									inputs.update({"start_positions": start_positions, "end_positions": end_positions} )

					return inputs


