
from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


lowercase__							:							Any													=	logging.get_logger(__name__)


lowercase__							:							Optional[int]													=	TypeVar("DatasetType", Dataset, IterableDataset)




def 		A_       (							snake_case							:      List[DatasetType]       ,					snake_case							:      Optional[List[float]] = None       ,					snake_case							:      Optional[int] = None       ,					snake_case							:      Optional[DatasetInfo] = None       ,					snake_case							:      Optional[NamedSplit] = None       ,					snake_case							:      Literal["first_exhausted", "all_exhausted"] = "first_exhausted"       ,					)					->			DatasetType:



						'''simple docstring'''

						from .arrow_dataset import Dataset
						from .iterable_dataset import IterableDataset

						if not datasets:
												raise ValueError('''Unable to interleave an empty list of datasets.'''       )
						for i, dataset in enumerate(__A       ):
												if not isinstance(__A       ,					(Dataset, IterableDataset)       ):
																		if isinstance(__A       ,					(DatasetDict, IterableDatasetDict)       ):
																								if not dataset:
																														raise ValueError(
																														    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
																														    '''is an empty dataset dictionary.'''       )
																								raise ValueError(
																								    f"Dataset at position {i} has at least one split: {list(__A       )}\n"
																								    f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__A       )       )}']"       )
																		raise ValueError(
																		    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__A       ).__name__}."       )
												if i == 0:
																		__UpperCamelCase     ,							__UpperCamelCase        =	(
																		    (Dataset, IterableDataset) if isinstance(__A       ,					__A       ) else (IterableDataset, Dataset)
																		)
												elif not isinstance(__A       ,					__A       ):
																		raise ValueError(
																		    f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."       )
						if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
												raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy."       )
						if dataset_type is Dataset:
												return _interleave_map_style_datasets(
												    __A       ,					__A       ,					__A       ,					info=__A       ,					split=__A       ,					stopping_strategy=__A       )
						else:
												return _interleave_iterable_datasets(
												    __A       ,					__A       ,					__A       ,					info=__A       ,					split=__A       ,					stopping_strategy=__A       )




def 		A_       (							snake_case							:      List[DatasetType]       ,					snake_case							:      Optional[DatasetInfo] = None       ,					snake_case							:      Optional[NamedSplit] = None       ,					snake_case							:      int = 0       ,					)					->			DatasetType:



						'''simple docstring'''

						if not dsets:
												raise ValueError('''Unable to concatenate an empty list of datasets.'''       )
						for i, dataset in enumerate(__A       ):
												if not isinstance(__A       ,					(Dataset, IterableDataset)       ):
																		if isinstance(__A       ,					(DatasetDict, IterableDatasetDict)       ):
																								if not dataset:
																														raise ValueError(
																														    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
																														    '''is an empty dataset dictionary.'''       )
																								raise ValueError(
																								    f"Dataset at position {i} has at least one split: {list(__A       )}\n"
																								    f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__A       )       )}']"       )
																		raise ValueError(
																		    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__A       ).__name__}."       )
												if i == 0:
																		__UpperCamelCase     ,							__UpperCamelCase        =	(
																		    (Dataset, IterableDataset) if isinstance(__A       ,					__A       ) else (IterableDataset, Dataset)
																		)
												elif not isinstance(__A       ,					__A       ):
																		raise ValueError(
																		    f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."       )
						if dataset_type is Dataset:
												return _concatenate_map_style_datasets(__A       ,					info=__A       ,					split=__A       ,					axis=__A       )
						else:
												return _concatenate_iterable_datasets(__A       ,					info=__A       ,					split=__A       ,					axis=__A       )










import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_ : List[Any]    		=       data_utils.TransfoXLTokenizer
snake_case_ : int    		=       data_utils.TransfoXLCorpus
snake_case_ : List[Any]    		=       data_utils
snake_case_ : int    		=       data_utils


def    A						(__A   :    Dict   ,				__A   :    List[Any]   ,				__A   :    Union[str, Any]   ,				__A   :    Tuple       )   ->      Union[str, Any]:







     """simple docstring"""


     if transfo_xl_dataset_file:
          # Convert a pre-processed corpus (see original TensorFlow repo)
          with open(__A   ,				'''rb'''       ) as fp:
               UpperCAmelCase_          =							pickle.load(__A   ,				encoding='''latin1'''       )
          # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
          print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""       )
          UpperCAmelCase_          =							corpus.vocab.__dict__
          torch.save(__A   ,				__A       )

          UpperCAmelCase_          =							corpus.__dict__
          corpus_dict_no_vocab.pop('''vocab'''   ,				__A       )
          UpperCAmelCase_          =							pytorch_dump_folder_path + '''/''' + CORPUS_NAME
          print(F"""Save dataset to {pytorch_dataset_dump_path}"""       )
          torch.save(__A   ,				__A       )

     if tf_checkpoint_path:
          # Convert a pre-trained TensorFlow model
          UpperCAmelCase_          =							os.path.abspath(__A       )
          UpperCAmelCase_          =							os.path.abspath(__A       )

          print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""       )
          # Initialise PyTorch model
          if transfo_xl_config_file == "":
               UpperCAmelCase_          =							TransfoXLConfig()
          else:
               UpperCAmelCase_          =							TransfoXLConfig.from_json_file(__A       )
          print(F"""Building PyTorch model from configuration: {config}"""       )
          UpperCAmelCase_          =							TransfoXLLMHeadModel(__A       )

          UpperCAmelCase_          =							load_tf_weights_in_transfo_xl(__A   ,				__A   ,				__A       )
          # Save pytorch-model
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          UpperCAmelCase_          =							os.path.join(__A   ,				__A       )
          print(F"""Save PyTorch model to {os.path.abspath(__A       )}"""       )
          torch.save(model.state_dict()   ,				__A       )
          print(F"""Save configuration file to {os.path.abspath(__A       )}"""       )
          with open(__A   ,				'''w'''   ,				encoding='''utf-8'''       ) as f:
               f.write(config.to_json_string()       )


if __name__ == "__main__":
    snake_case_ : List[str]    		=       argparse.ArgumentParser()
    parser.add_argument(
        "--pytorch_dump_folder_path",
        default=None,
        type=str,
        required=True,
        help="Path to the folder to store the PyTorch model or dataset/vocab.",
    )
    parser.add_argument(
        "--tf_checkpoint_path",
        default="",
        type=str,
        help="An optional path to a TensorFlow checkpoint path to be converted.",
    )
    parser.add_argument(
        "--transfo_xl_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained BERT model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--transfo_xl_dataset_file",
        default="",
        type=str,
        help="An optional dataset file to be converted in a vocabulary.",
    )
    snake_case_ : int    		=       parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )






