






import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402


a		:       List[str]			     = {
    "return_dict": False,
    "output_hidden_states": True,
    "output_attentions": True,
    "torchscript": True,
    "torch_dtype": "float16",
    "use_bfloat16": True,
    "tf_legacy_loss": True,
    "pruned_heads": {"a": 1},
    "tie_word_embeddings": False,
    "is_decoder": True,
    "cross_attention_hidden_size": 128,
    "add_cross_attention": True,
    "tie_encoder_decoder": True,
    "max_length": 50,
    "min_length": 3,
    "do_sample": True,
    "early_stopping": True,
    "num_beams": 3,
    "num_beam_groups": 3,
    "diversity_penalty": 0.5,
    "temperature": 2.0,
    "top_k": 10,
    "top_p": 0.7,
    "typical_p": 0.2,
    "repetition_penalty": 0.8,
    "length_penalty": 0.8,
    "no_repeat_ngram_size": 5,
    "encoder_no_repeat_ngram_size": 5,
    "bad_words_ids": [1, 2, 3],
    "num_return_sequences": 3,
    "chunk_size_feed_forward": 5,
    "output_scores": True,
    "return_dict_in_generate": True,
    "forced_bos_token_id": 2,
    "forced_eos_token_id": 3,
    "remove_invalid_values": True,
    "architectures": ["BertModel"],
    "finetuning_task": "translation",
    "id2label": {0: "label"},
    "label2id": {"label": "0"},
    "tokenizer_class": "BertTokenizerFast",
    "prefix": "prefix",
    "bos_token_id": 6,
    "pad_token_id": 7,
    "eos_token_id": 8,
    "sep_token_id": 9,
    "decoder_start_token_id": 10,
    "exponential_decay_length_penalty": (5, 1.01),
    "suppress_tokens": [0, 1],
    "begin_suppress_tokens": 2,
    "task_specific_params": {"translation": "some_params"},
    "problem_type": "regression",
}
@is_staging_test
class     a    ( unittest.TestCase   ):



      """simple docstring"""


      @classmethod
      def 	UpperCAmelCase      (      cls					:			Optional[Any]					)							->      str:
        __UpperCAmelCase							:	Tuple           =  TOKEN
        HfFolder.save_token(UpperCAmelCase_					)
      @classmethod
      def 	UpperCAmelCase      (      cls					:			List[str]					)							->      str:



        try:
          delete_repo(token=cls._token		,      repo_id="""test-config"""					)
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token		,      repo_id="""valid_org/test-config-org"""					)
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token		,      repo_id="""test-dynamic-config"""					)
        except HTTPError:
          pass
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Union[str, Any]:
        __UpperCAmelCase							:	List[Any]           =  BertConfig(
            vocab_size=99		,      hidden_size=32		,      num_hidden_layers=5		,      num_attention_heads=4		,      intermediate_size=37					)
        config.push_to_hub("""test-config"""		,      use_auth_token=self._token					)

        __UpperCAmelCase							:	Dict           =  BertConfig.from_pretrained(f"""{USER}/test-config"""					)
        for k, v in config.to_dict().items():
          if k != "transformers_version":
            self.assertEqual(UpperCAmelCase_		,      getattr(UpperCAmelCase_		,      UpperCAmelCase_					)					)

             # Reset repo
        delete_repo(token=self._token		,      repo_id="""test-config"""					)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          config.save_pretrained(UpperCAmelCase_		,      repo_id="""test-config"""		,      push_to_hub=UpperCAmelCase_		,      use_auth_token=self._token					)

        __UpperCAmelCase							:	Optional[int]           =  BertConfig.from_pretrained(f"""{USER}/test-config"""					)
        for k, v in config.to_dict().items():
          if k != "transformers_version":
            self.assertEqual(UpperCAmelCase_		,      getattr(UpperCAmelCase_		,      UpperCAmelCase_					)					)
      def 	UpperCAmelCase      (      self					:			Any					)							->      Tuple:
        __UpperCAmelCase							:	Optional[int]           =  BertConfig(
            vocab_size=99		,      hidden_size=32		,      num_hidden_layers=5		,      num_attention_heads=4		,      intermediate_size=37					)
        config.push_to_hub("""valid_org/test-config-org"""		,      use_auth_token=self._token					)

        __UpperCAmelCase							:	Optional[int]           =  BertConfig.from_pretrained("""valid_org/test-config-org"""					)
        for k, v in config.to_dict().items():
          if k != "transformers_version":
            self.assertEqual(UpperCAmelCase_		,      getattr(UpperCAmelCase_		,      UpperCAmelCase_					)					)

             # Reset repo
        delete_repo(token=self._token		,      repo_id="""valid_org/test-config-org"""					)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          config.save_pretrained(
              UpperCAmelCase_		,      repo_id="""valid_org/test-config-org"""		,      push_to_hub=UpperCAmelCase_		,      use_auth_token=self._token					)

        __UpperCAmelCase							:	List[str]           =  BertConfig.from_pretrained("""valid_org/test-config-org"""					)
        for k, v in config.to_dict().items():
          if k != "transformers_version":
            self.assertEqual(UpperCAmelCase_		,      getattr(UpperCAmelCase_		,      UpperCAmelCase_					)					)




      def 	UpperCAmelCase      (      self					:			Optional[Any]					)							->      Optional[int]:
        CustomConfig.register_for_auto_class()
        __UpperCAmelCase							:	Optional[Any]           =  CustomConfig(attribute=42					)

        config.push_to_hub("""test-dynamic-config"""		,      use_auth_token=self._token					)

        # This has added the proper auto_map field to the config
        self.assertDictEqual(config.auto_map		,      {"""AutoConfig""": """custom_configuration.CustomConfig"""}					)

        __UpperCAmelCase							:	List[str]           =  AutoConfig.from_pretrained(f"""{USER}/test-dynamic-config"""		,      trust_remote_code=UpperCAmelCase_					)
        # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
        self.assertEqual(new_config.__class__.__name__		,      """CustomConfig"""					)
        self.assertEqual(new_config.attribute		,      42					)


class     a    ( unittest.TestCase   ):



      """simple docstring"""


      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      Dict:
        __UpperCAmelCase							:	Tuple           =  GPTaConfig()

        # attempt to modify each of int/float/bool/str config records and verify they were updated
        __UpperCAmelCase							:	Tuple           =  c.n_embd + 1  # int
        __UpperCAmelCase							:	Optional[int]           =  c.resid_pdrop + 1.0  # float
        __UpperCAmelCase							:	Dict           =  not c.scale_attn_weights  # bool
        __UpperCAmelCase							:	Any           =  c.summary_type + "foo"  # str
        c.update_from_string(
            f"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""					)
        self.assertEqual(UpperCAmelCase_		,      c.n_embd		,      """mismatch for key: n_embd"""					)
        self.assertEqual(UpperCAmelCase_		,      c.resid_pdrop		,      """mismatch for key: resid_pdrop"""					)
        self.assertEqual(UpperCAmelCase_		,      c.scale_attn_weights		,      """mismatch for key: scale_attn_weights"""					)
        self.assertEqual(UpperCAmelCase_		,      c.summary_type		,      """mismatch for key: summary_type"""					)
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      Union[str, Any]:
        __UpperCAmelCase							:	Union[str, Any]           =  PretrainedConfig()
        __UpperCAmelCase							:	str           =  [key for key in base_config.__dict__ if key not in config_common_kwargs]
        # If this part of the test fails, you have arguments to addin config_common_kwargs above.
        self.assertListEqual(
            UpperCAmelCase_		,      ["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]					)
        __UpperCAmelCase							:	int           =  [key for key, value in config_common_kwargs.items() if value == getattr(UpperCAmelCase_		,      UpperCAmelCase_					)]
        if len(UpperCAmelCase_					) > 0:
          raise ValueError(
              """The following keys are set with the default values in"""
              """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
              f""" {", ".join(UpperCAmelCase_					)}."""					)
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      Any:



        with self.assertRaises(UpperCAmelCase_					):
          # config is in subfolder, the following should not work without specifying the subfolder
          __UpperCAmelCase							:	Tuple           =  BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""					)

        __UpperCAmelCase							:	Union[str, Any]           =  BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""		,      subfolder="""bert"""					)

        self.assertIsNotNone(UpperCAmelCase_					)
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Union[str, Any]:
        __UpperCAmelCase							:	Tuple           =  mock.Mock()
        __UpperCAmelCase							:	Tuple           =  500
        __UpperCAmelCase							:	Union[str, Any]           =  {}
        __UpperCAmelCase							:	str           =  HTTPError
        __UpperCAmelCase							:	Tuple           =  {}

        # Download this model to make sure it's in the cache.
        __UpperCAmelCase							:	str           =  BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)

        # Under the mock environment we get a 500 error when trying to reach the model.
        with mock.patch("""requests.Session.request"""		,      return_value=UpperCAmelCase_					) as mock_head:
          __UpperCAmelCase							:	List[str]           =  BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)
          # This check we did call the fake head request
          mock_head.assert_called()
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      Union[str, Any]:
        __UpperCAmelCase							:	Optional[Any]           =  BertConfig.from_pretrained(
            """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""					)
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      int:
        __UpperCAmelCase							:	Union[str, Any]           =  AutoConfig.from_pretrained("""bert-base-cased"""					)
        __UpperCAmelCase							:	Any           =  ["config.4.0.0.json"]

        with tempfile.TemporaryDirectory() as tmp_dir:
          configuration.save_pretrained(UpperCAmelCase_					)
          __UpperCAmelCase							:	List[Any]           =  2
          json.dump(configuration.to_dict()		,      open(os.path.join(UpperCAmelCase_		,      """config.4.0.0.json"""					)		,      """w"""					)					)

          # This should pick the new configuration file as the version of Transformers is > 4.0.0
          __UpperCAmelCase							:	Optional[Any]           =  AutoConfig.from_pretrained(UpperCAmelCase_					)
          self.assertEqual(new_configuration.hidden_size		,      2					)

          # Will need to be adjusted if we reach v42 and this test is still here.
          # Should pick the old configuration file as the version of Transformers is < 4.42.0
          __UpperCAmelCase							:	Any           =  ["config.42.0.0.json"]
          __UpperCAmelCase							:	List[Any]           =  768
          configuration.save_pretrained(UpperCAmelCase_					)
          shutil.move(os.path.join(UpperCAmelCase_		,      """config.4.0.0.json"""					)		,      os.path.join(UpperCAmelCase_		,      """config.42.0.0.json"""					)					)
          __UpperCAmelCase							:	Optional[int]           =  AutoConfig.from_pretrained(UpperCAmelCase_					)
          self.assertEqual(new_configuration.hidden_size		,      768					)




      def 	UpperCAmelCase      (      self					:			Dict					)							->      Optional[Any]:
        __UpperCAmelCase							:	List[str]           =  "hf-internal-testing/test-two-configs"

        import transformers as new_transformers

        __UpperCAmelCase							:	Dict           =  "v4.0.0"
        __UpperCAmelCase							:	Optional[int]           =  new_transformers.models.auto.AutoConfig.from_pretrained(
            UpperCAmelCase_		,      return_unused_kwargs=UpperCAmelCase_					)
        self.assertEqual(new_configuration.hidden_size		,      2					)
        # This checks `_configuration_file` ia not kept in the kwargs by mistake.
        self.assertDictEqual(UpperCAmelCase_		,      {}					)

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        __UpperCAmelCase							:	str           =  "v3.0.0"
        __UpperCAmelCase							:	Tuple           =  old_transformers.models.auto.AutoConfig.from_pretrained(UpperCAmelCase_					)
        self.assertEqual(old_configuration.hidden_size		,      768					)









import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def      lowerCAmelCase_ (       __a	,    __a		)     ->		Optional[Any]:







  """simple docstring"""





  assert isinstance(__a	,    __a		)
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("keep_in_memory"	,    [False, True]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		List[Any]:







  """simple docstring"""





  lowerCamelCase__:      Any			  =tmp_path / "cache"
  lowerCamelCase__:      Optional[int]			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCamelCase__:      int			  =ParquetDatasetReader(__a	,    cache_dir=__a	,    keep_in_memory=__a		).read()
  _check_parquet_dataset(__a	,    __a		)





@pytest.mark.parametrize(
    "features"	,    [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,    )
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		List[Any]:







  """simple docstring"""





  lowerCamelCase__:      int			  =tmp_path / "cache"
  lowerCamelCase__:      Tuple			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCamelCase__:      Union[str, Any]			  =features.copy() if features else default_expected_features
  lowerCamelCase__:      Union[str, Any]			  =(
      Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
  )
  lowerCamelCase__:      int			  =ParquetDatasetReader(__a	,    features=__a	,    cache_dir=__a		).read()
  _check_parquet_dataset(__a	,    __a		)





@pytest.mark.parametrize("split"	,    [None, NamedSplit("train"		), "train", "test"]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		Any:







  """simple docstring"""





  lowerCamelCase__:      Union[str, Any]			  =tmp_path / "cache"
  lowerCamelCase__:      Dict			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCamelCase__:      Optional[int]			  =ParquetDatasetReader(__a	,    cache_dir=__a	,    split=__a		).read()
  _check_parquet_dataset(__a	,    __a		)
  assert dataset.split == split if split else "train"





@pytest.mark.parametrize("path_type"	,    [str, list]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		Dict:







  """simple docstring"""





  if issubclass(__a	,    __a		):
    lowerCamelCase__:      str			  =parquet_path
  elif issubclass(__a	,    __a		):
    lowerCamelCase__:      str			  =[parquet_path]
  lowerCamelCase__:      Optional[Any]			  =tmp_path / "cache"
  lowerCamelCase__:      Any			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCamelCase__:      Optional[int]			  =ParquetDatasetReader(__a	,    cache_dir=__a		).read()
  _check_parquet_dataset(__a	,    __a		)





def      lowerCAmelCase_ (       __a	,    __a	,    __a=("train",)		)     ->		Union[str, Any]:







  """simple docstring"""





  assert isinstance(__a	,    __a		)
  for split in splits:
    lowerCamelCase__:      Optional[Any]			  =dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("keep_in_memory"	,    [False, True]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		List[Any]:







  """simple docstring"""





  lowerCamelCase__:      Any			  =tmp_path / "cache"
  lowerCamelCase__:      str			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCamelCase__:      List[str]			  =ParquetDatasetReader(
        {"train": parquet_path}	,    cache_dir=__a	,    keep_in_memory=__a		).read()
  _check_parquet_datasetdict(__a	,    __a		)





@pytest.mark.parametrize(
    "features"	,    [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,    )
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		List[Any]:







  """simple docstring"""





  lowerCamelCase__:      List[Any]			  =tmp_path / "cache"
  lowerCamelCase__:      Any			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCamelCase__:      int			  =features.copy() if features else default_expected_features
  lowerCamelCase__:      Union[str, Any]			  =(
      Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
  )
  lowerCamelCase__:      Union[str, Any]			  =ParquetDatasetReader({"train": parquet_path}	,    features=__a	,    cache_dir=__a		).read()
  _check_parquet_datasetdict(__a	,    __a		)





@pytest.mark.parametrize("split"	,    [None, NamedSplit("train"		), "train", "test"]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		List[str]:







  """simple docstring"""





  if split:
    lowerCamelCase__:      Union[str, Any]			  ={split: parquet_path}
  else:
    lowerCamelCase__:      int			  ="train"
    lowerCamelCase__:      Union[str, Any]			  ={"train": parquet_path, "test": parquet_path}
  lowerCamelCase__:      int			  =tmp_path / "cache"
  lowerCamelCase__:      Union[str, Any]			  ={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCamelCase__:      Optional[Any]			  =ParquetDatasetReader(__a	,    cache_dir=__a		).read()
  _check_parquet_datasetdict(__a	,    __a	,    splits=list(path.keys()		)		)
  assert all(dataset[split].split == split for split in path.keys()		)





def      lowerCAmelCase_ (       __a	,    __a		)     ->		Tuple:







  """simple docstring"""





  lowerCamelCase__:      Tuple			  =ParquetDatasetWriter(__a	,    tmp_path / "foo.parquet"		)
  assert writer.write() > 0
  lowerCamelCase__:      Tuple			  =pq.ParquetFile(tmp_path / "foo.parquet"		)
  lowerCamelCase__:      Optional[int]			  =pf.read()
  assert dataset.data.table == output_table





def      lowerCAmelCase_ (       __a	,    __a		)     ->		List[Any]:







  """simple docstring"""





  lowerCamelCase__:      List[str]			  =str(shared_datadir / "test_image_rgb.jpg"		)
  lowerCamelCase__:      Union[str, Any]			  ={"image": [image_path]}
  lowerCamelCase__:      int			  =Features({"image": Image()}		)
  lowerCamelCase__:      Tuple			  =Dataset.from_dict(__a	,    features=__a		)
  lowerCamelCase__:      Optional[int]			  =ParquetDatasetWriter(__a	,    tmp_path / "foo.parquet"		)
  assert writer.write() > 0

  lowerCamelCase__:      Optional[Any]			  =Dataset.from_parquet(str(tmp_path / "foo.parquet"		)		)
  assert dataset.features == reloaded_dataset.features

  lowerCamelCase__:      List[str]			  =ParquetDatasetReader(str(tmp_path / "foo.parquet"		)	,    streaming=__a		).read()
  assert dataset.features == reloaded_iterable_dataset.features






@pytest.mark.parametrize(
    "feature, expected"	,    [
        (Features({"foo": Value("int32"		)}		), None),
        (Features({"image": Image(), "foo": Value("int32"		)}		), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"nested": Sequence(Audio()		)}		), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]	,    )
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Any:







  """simple docstring"""





  assert get_writer_batch_size(__a		) == expected

