




import os

import pytest

from datasets import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
)


snake_case_ : str    		=       pytest.mark.integration


@pytest.mark.parametrize('''path'''   ,				['''paws''', '''csv''']       )
def    A						(__A   :    Union[str, Any]   ,				__A   :    Dict       )   ->      Optional[Any]:







     """simple docstring"""


     inspect_dataset(__A   ,				__A       )
     UpperCAmelCase_          =							path + '''.py'''
     assert script_name in os.listdir(__A       )
     assert "__pycache__" not in os.listdir(__A       )


@pytest.mark.filterwarnings('''ignore:inspect_metric is deprecated:FutureWarning'''       )
@pytest.mark.filterwarnings('''ignore:metric_module_factory is deprecated:FutureWarning'''       )
@pytest.mark.parametrize('''path'''   ,				['''accuracy''']       )
def    A						(__A   :    Optional[Any]   ,				__A   :    List[str]       )   ->      Tuple:







     """simple docstring"""


     inspect_metric(__A   ,				__A       )
     UpperCAmelCase_          =							path + '''.py'''
     assert script_name in os.listdir(__A       )
     assert "__pycache__" not in os.listdir(__A       )


@pytest.mark.parametrize(
    '''path, config_name, expected_splits'''   ,				[
        ('''squad''', '''plain_text''', ['''train''', '''validation''']),
        ('''dalle-mini/wit''', '''dalle-mini--wit''', ['''train''']),
        ('''paws''', '''labeled_final''', ['''train''', '''test''', '''validation''']),
    ]   ,				)
def    A						(__A   :    Optional[Any]   ,				__A   :    int   ,				__A   :    Optional[int]       )   ->      Tuple:







     """simple docstring"""


     UpperCAmelCase_          =							get_dataset_config_info(__A   ,				config_name=__A       )
     assert info.config_name == config_name
     assert list(info.splits.keys()       ) == expected_splits


@pytest.mark.parametrize(
    '''path, config_name, expected_exception'''   ,				[
        ('''paws''', None, ValueError),
    ]   ,				)
def    A						(__A   :    List[Any]   ,				__A   :    Optional[int]   ,				__A   :    List[str]       )   ->      Optional[int]:







     """simple docstring"""


     with pytest.raises(__A       ):
          get_dataset_config_info(__A   ,				config_name=__A       )


@pytest.mark.parametrize(
    '''path, expected'''   ,				[
        ('''squad''', '''plain_text'''),
        ('''acronym_identification''', '''default'''),
        ('''lhoestq/squad''', '''plain_text'''),
        ('''lhoestq/test''', '''default'''),
        ('''lhoestq/demo1''', '''lhoestq--demo1'''),
        ('''dalle-mini/wit''', '''dalle-mini--wit'''),
    ]   ,				)
def    A						(__A   :    Any   ,				__A   :    Dict       )   ->      List[str]:







     """simple docstring"""


     UpperCAmelCase_          =							get_dataset_config_names(__A       )
     assert expected in config_names


@pytest.mark.parametrize(
    '''path, expected_configs, expected_splits_in_first_config'''   ,				[
        ('''squad''', ['''plain_text'''], ['''train''', '''validation''']),
        ('''dalle-mini/wit''', ['''dalle-mini--wit'''], ['''train''']),
        ('''paws''', ['''labeled_final''', '''labeled_swap''', '''unlabeled_final'''], ['''train''', '''test''', '''validation''']),
    ]   ,				)
def    A						(__A   :    List[str]   ,				__A   :    List[str]   ,				__A   :    int       )   ->      Any:







     """simple docstring"""


     UpperCAmelCase_          =							get_dataset_infos(__A       )
     assert list(infos.keys()       ) == expected_configs
     UpperCAmelCase_          =							expected_configs[0]
     assert expected_config in infos
     UpperCAmelCase_          =							infos[expected_config]
     assert info.config_name == expected_config
     assert list(info.splits.keys()       ) == expected_splits_in_first_config


@pytest.mark.parametrize(
    '''path, expected_config, expected_splits'''   ,				[
        ('''squad''', '''plain_text''', ['''train''', '''validation''']),
        ('''dalle-mini/wit''', '''dalle-mini--wit''', ['''train''']),
        ('''paws''', '''labeled_final''', ['''train''', '''test''', '''validation''']),
    ]   ,				)
def    A						(__A   :    Optional[Any]   ,				__A   :    List[Any]   ,				__A   :    Tuple       )   ->      str:







     """simple docstring"""


     UpperCAmelCase_          =							get_dataset_infos(__A       )
     assert expected_config in infos
     UpperCAmelCase_          =							infos[expected_config]
     assert info.config_name == expected_config
     assert list(info.splits.keys()       ) == expected_splits


@pytest.mark.parametrize(
    '''path, config_name, expected_exception'''   ,				[
        ('''paws''', None, ValueError),
    ]   ,				)
def    A						(__A   :    str   ,				__A   :    List[str]   ,				__A   :    Union[str, Any]       )   ->      Optional[Any]:







     """simple docstring"""


     with pytest.raises(__A       ):
          get_dataset_split_names(__A   ,				config_name=__A       )












import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_ : List[Any]    		=       data_utils.TransfoXLTokenizer
snake_case_ : int    		=       data_utils.TransfoXLCorpus
snake_case_ : List[Any]    		=       data_utils
snake_case_ : int    		=       data_utils


def    A						(__A   :    Dict   ,				__A   :    List[Any]   ,				__A   :    Union[str, Any]   ,				__A   :    Tuple       )   ->      Union[str, Any]:







     """simple docstring"""


     if transfo_xl_dataset_file:
          # Convert a pre-processed corpus (see original TensorFlow repo)
          with open(__A   ,				'''rb'''       ) as fp:
               UpperCAmelCase_          =							pickle.load(__A   ,				encoding='''latin1'''       )
          # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
          print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""       )
          UpperCAmelCase_          =							corpus.vocab.__dict__
          torch.save(__A   ,				__A       )

          UpperCAmelCase_          =							corpus.__dict__
          corpus_dict_no_vocab.pop('''vocab'''   ,				__A       )
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + CORPUS_NAME
          print(F"""Save dataset to {pytorch_dataset_dump_path}"""       )
          torch.save(__A   ,				__A       )

     if tf_checkpoint_path:
          # Convert a pre-trained TensorFlow model
          UpperCAmelCase_          =							os.path.abspath(__A       )
          UpperCAmelCase_          =							os.path.abspath(__A       )

          print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""       )
          # Initialise PyTorch model
          if transfo_xl_config_file == "":
               UpperCAmelCase_          =							TransfoXLConfig()
          else:
               UpperCAmelCase_          =							TransfoXLConfig.from_json_file(__A       )
          print(F"""Building PyTorch model from configuration: {config}"""       )
          UpperCAmelCase_          =							TransfoXLLMHeadModel(__A       )

          UpperCAmelCase_          =							load_tf_weights_in_transfo_xl(__A   ,				__A   ,				__A       )
          # Save pytorch-model
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          print(F"""Save PyTorch model to {os.path.abspath(__A       )}"""       )
          torch.save(model.state_dict()   ,				__A       )
          print(F"""Save configuration file to {os.path.abspath(__A       )}"""       )
          with open(__A   ,				'''w'''   ,				encoding='''utf-8'''       ) as f:
               f.write(config.to_json_string()       )


if __name__ == "__main__":
    snake_case_ : List[str]    		=       argparse.ArgumentParser()
    parser.add_argument(
        "--pytorch_dump_folder_path",
        default=None,
        type=str,
        required=True,
        help="Path to the folder to store the PyTorch model or dataset/vocab.",
    )
    parser.add_argument(
        "--tf_checkpoint_path",
        default="",
        type=str,
        help="An optional path to a TensorFlow checkpoint path to be converted.",
    )
    parser.add_argument(
        "--transfo_xl_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained BERT model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--transfo_xl_dataset_file",
        default="",
        type=str,
        help="An optional dataset file to be converted in a vocabulary.",
    )
    snake_case_ : int    		=       parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )






