




import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test







class __snake_case			(			unittest.TestCase		):







   @parameterized.expand([(None,), ('''foo.json''',)])
   def lowerCamelCase						(      self		:						Optional[int]					,						_snake_case		:						Dict):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig(
            do_sample=_snake_case					,						temperature=0.7					,						length_penalty=1.0					,						bad_words_ids=[[1, 2, 3], [4, 5]]					,						)
        with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(_snake_case					,						config_name=_snake_case)
             UpperCAmelCase_          =							GenerationConfig.from_pretrained(_snake_case					,						config_name=_snake_case)

        # Checks parameters that were specified
        self.assertEqual(loaded_config.do_sample					,						_snake_case)
        self.assertEqual(loaded_config.temperature					,						0.7)
        self.assertEqual(loaded_config.length_penalty					,						1.0)
        self.assertEqual(loaded_config.bad_words_ids					,						[[1, 2, 3], [4, 5]])

        # Checks parameters that were not specified (defaults)
        self.assertEqual(loaded_config.top_k					,						50)
        self.assertEqual(loaded_config.max_length					,						20)
        self.assertEqual(loaded_config.max_time					,						_snake_case)







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							AutoConfig.from_pretrained('''gpt2''')
        UpperCAmelCase_          =							GenerationConfig.from_model_config(_snake_case)
        UpperCAmelCase_          =							GenerationConfig()

        # The generation config has loaded a few non-default parameters from the model config
        self.assertNotEqual(_snake_case					,						_snake_case)

        # One of those parameters is eos_token_id -- check if it matches
        self.assertNotEqual(generation_config_from_model.eos_token_id					,						default_generation_config.eos_token_id)
        self.assertEqual(generation_config_from_model.eos_token_id					,						model_config.eos_token_id)







   def lowerCamelCase						(      self		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig()
        UpperCAmelCase_          =							{
            '''max_new_tokens''': 1024,
            '''foo''': '''bar''',
        }
        UpperCAmelCase_          =							copy.deepcopy(_snake_case)
        UpperCAmelCase_          =							generation_config.update(**_snake_case)

        # update_kwargs was not modified (no side effects)
        self.assertEqual(_snake_case					,						_snake_case)

        # update_kwargs was used to update the config on valid attributes
        self.assertEqual(generation_config.max_new_tokens					,						1024)

        # `.update()` returns a dictionary of unused kwargs
        self.assertEqual(_snake_case					,						{'''foo''': '''bar'''})







   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig()
        UpperCAmelCase_          =							'''bar'''

        with tempfile.TemporaryDirectory('''test-generation-config''') as tmp_dir:
             generation_config.save_pretrained(_snake_case)

             UpperCAmelCase_          =							GenerationConfig.from_pretrained(_snake_case)
        # update_kwargs was used to update the config on valid attributes
        self.assertEqual(new_config.foo					,						'''bar''')

        UpperCAmelCase_          =							GenerationConfig.from_model_config(_snake_case)
        assert not hasattr(_snake_case					,						'''foo''')  # no new kwargs should be initialized if from config







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig()
        self.assertEqual(default_config.temperature					,						1.0)
        self.assertEqual(default_config.do_sample					,						_snake_case)
        self.assertEqual(default_config.num_beams					,						1)

        UpperCAmelCase_          =							GenerationConfig(
            do_sample=_snake_case					,						temperature=0.7					,						length_penalty=1.0					,						bad_words_ids=[[1, 2, 3], [4, 5]]					,						)
        self.assertEqual(config.temperature					,						0.7)
        self.assertEqual(config.do_sample					,						_snake_case)
        self.assertEqual(config.num_beams					,						1)

        with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(_snake_case)
             UpperCAmelCase_          =							GenerationConfig.from_pretrained(_snake_case					,						temperature=1.0)

        self.assertEqual(loaded_config.temperature					,						1.0)
        self.assertEqual(loaded_config.do_sample					,						_snake_case)
        self.assertEqual(loaded_config.num_beams					,						1)  # default value







@is_staging_test
class __snake_case			(			unittest.TestCase		):







   @classmethod
   def lowerCamelCase						(      cls		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							TOKEN
        HfFolder.save_token(_snake_case)







   @classmethod
   def lowerCamelCase						(      cls		:						Tuple):

        """simple docstring"""


        try:
             delete_repo(token=cls._token					,						repo_id='''test-generation-config''')
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token					,						repo_id='''valid_org/test-generation-config-org''')
        except HTTPError:
             pass







   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig(
            do_sample=_snake_case					,						temperature=0.7					,						length_penalty=1.0					,						)
        config.push_to_hub('''test-generation-config'''					,						use_auth_token=self._token)

        UpperCAmelCase_          =							GenerationConfig.from_pretrained(F"""{USER}/test-generation-config""")
        for k, v in config.to_dict().items():
             if k != "transformers_version":
                  self.assertEqual(_snake_case					,						getattr(_snake_case					,						_snake_case))

        # Reset repo
        delete_repo(token=self._token					,						repo_id='''test-generation-config''')

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(
                 _snake_case					,						repo_id='''test-generation-config'''					,						push_to_hub=_snake_case					,						use_auth_token=self._token)

        UpperCAmelCase_          =							GenerationConfig.from_pretrained(F"""{USER}/test-generation-config""")
        for k, v in config.to_dict().items():
             if k != "transformers_version":
                  self.assertEqual(_snake_case					,						getattr(_snake_case					,						_snake_case))







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							GenerationConfig(
            do_sample=_snake_case					,						temperature=0.7					,						length_penalty=1.0					,						)
        config.push_to_hub('''valid_org/test-generation-config-org'''					,						use_auth_token=self._token)

        UpperCAmelCase_          =							GenerationConfig.from_pretrained('''valid_org/test-generation-config-org''')
        for k, v in config.to_dict().items():
             if k != "transformers_version":
                  self.assertEqual(_snake_case					,						getattr(_snake_case					,						_snake_case))

        # Reset repo
        delete_repo(token=self._token					,						repo_id='''valid_org/test-generation-config-org''')

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(
                 _snake_case					,						repo_id='''valid_org/test-generation-config-org'''					,						push_to_hub=_snake_case					,						use_auth_token=self._token)

        UpperCAmelCase_          =							GenerationConfig.from_pretrained('''valid_org/test-generation-config-org''')
        for k, v in config.to_dict().items():
             if k != "transformers_version":
                  self.assertEqual(_snake_case					,						getattr(_snake_case					,						_snake_case))












import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_ : List[Any]    		=       data_utils.TransfoXLTokenizer
snake_case_ : int    		=       data_utils.TransfoXLCorpus
snake_case_ : List[Any]    		=       data_utils
snake_case_ : int    		=       data_utils


def    A						(__A   :    Dict   ,				__A   :    List[Any]   ,				__A   :    Union[str, Any]   ,				__A   :    Tuple       )   ->      Union[str, Any]:







     """simple docstring"""


     if transfo_xl_dataset_file:
          # Convert a pre-processed corpus (see original TensorFlow repo)
          with open(__A   ,				'''rb'''       ) as fp:
               UpperCAmelCase_          =							pickle.load(__A   ,				encoding='''latin1'''       )
          # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
          print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""       )
          UpperCAmelCase_          =							corpus.vocab.__dict__
          torch.save(__A   ,				__A       )

          UpperCAmelCase_          =							corpus.__dict__
          corpus_dict_no_vocab.pop('''vocab'''   ,				__A       )
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + CORPUS_NAME
          print(F"""Save dataset to {pytorch_dataset_dump_path}"""       )
          torch.save(__A   ,				__A       )

     if tf_checkpoint_path:
          # Convert a pre-trained TensorFlow model
          UpperCAmelCase_          =							os.path.abspath(__A       )
          UpperCAmelCase_          =							os.path.abspath(__A       )

          print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""       )
          # Initialise PyTorch model
          if transfo_xl_config_file == "":
               UpperCAmelCase_          =							TransfoXLConfig()
          else:
               UpperCAmelCase_          =							TransfoXLConfig.from_json_file(__A       )
          print(F"""Building PyTorch model from configuration: {config}"""       )
          UpperCAmelCase_          =							TransfoXLLMHeadModel(__A       )

          UpperCAmelCase_          =							load_tf_weights_in_transfo_xl(__A   ,				__A   ,				__A       )
          # Save pytorch-model
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          print(F"""Save PyTorch model to {os.path.abspath(__A       )}"""       )
          torch.save(model.state_dict()   ,				__A       )
          print(F"""Save configuration file to {os.path.abspath(__A       )}"""       )
          with open(__A   ,				'''w'''   ,				encoding='''utf-8'''       ) as f:
               f.write(config.to_json_string()       )


if __name__ == "__main__":
    snake_case_ : List[str]    		=       argparse.ArgumentParser()
    parser.add_argument(
        "--pytorch_dump_folder_path",
        default=None,
        type=str,
        required=True,
        help="Path to the folder to store the PyTorch model or dataset/vocab.",
    )
    parser.add_argument(
        "--tf_checkpoint_path",
        default="",
        type=str,
        help="An optional path to a TensorFlow checkpoint path to be converted.",
    )
    parser.add_argument(
        "--transfo_xl_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained BERT model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--transfo_xl_dataset_file",
        default="",
        type=str,
        help="An optional dataset file to be converted in a vocabulary.",
    )
    snake_case_ : int    		=       parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )






