
'''simple docstring'''







import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowerCAmelCase_      =   logging.getLogger(__name__)







def   lowerCAmelCase(  ):







       '''simple docstring'''






       lowerCamelCase__				   =      argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."      )
       parser.add_argument(
           "--dataset_name"      , type=SCREAMING_SNAKE_CASE_      , default="wikitext"      , help="Name of the training. Explore datasets at: hf.co/datasets."      , )
       parser.add_argument(
           "--dataset_config"      , type=SCREAMING_SNAKE_CASE_      , default="wikitext-103-raw-v1"      , help="Configuration name of the dataset."      )
       parser.add_argument(
           "--batch_size"      , type=SCREAMING_SNAKE_CASE_      , default=1000      , help="Batch size during training."      , )
       parser.add_argument(
           "--vocab_size"      , type=SCREAMING_SNAKE_CASE_      , default=1_0048      , help="Size of the desired vocabulary."      , )
       parser.add_argument(
           "--limit"      , default=SCREAMING_SNAKE_CASE_      , type=SCREAMING_SNAKE_CASE_      , help="Limit the number of shards (used for debugging)."      , )
       parser.add_argument(
           "--export_to_hub"      , action="store_true"      , )

       lowerCamelCase__				   =      parser.parse_args()
       return args







def   lowerCAmelCase(  a__    :  Optional[int]      ):







       '''simple docstring'''






       lowerCamelCase__				   =      datasets.load_dataset(args.dataset_name      , args.dataset_config      , split="train"      )

       if args.limit is not None:
              lowerCamelCase__				   =      min(len(SCREAMING_SNAKE_CASE_      )      , args.limit      )
              lowerCamelCase__				   =      dataset.select(range(SCREAMING_SNAKE_CASE_      )      )
              logger.info(f"""Limiting the dataset to {args.limit} entries."""      )

       def batch_iterator():
              for i in range(0      , len(SCREAMING_SNAKE_CASE_      )      , args.batch_size      ):
                     yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
       lowerCamelCase__				   =      Tokenizer(Unigram()      )
       lowerCamelCase__				   =      normalizers.Sequence([normalizers.Replace("``"      , "\""      ), normalizers.Replace("\'\'"      , "\""      )]      )
       lowerCamelCase__				   =      pre_tokenizers.Metaspace()

       # Prepare the trainer.
       lowerCamelCase__				   =      UnigramTrainer(
           unk_token="<unk>"      , special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]      , vocab_size=args.vocab_size      , )

       logger.info("Training the tokenizer."      )
       tokenizer.train_from_iterator(batch_iterator()      , trainer=SCREAMING_SNAKE_CASE_      )
       logger.info("Tokenizer training complete!"      )

       lowerCamelCase__				   =      tokenizer.token_to_id("[CLS]"      )
       lowerCamelCase__				   =      tokenizer.token_to_id("[SEP]"      )
       lowerCamelCase__				   =      processors.TemplateProcessing(
           single="[CLS]:0 $A:0 [SEP]:0"      , pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"      , special_tokens=[
               ("[CLS]", cls_token_id),
               ("[SEP]", sep_token_id),
           ]      , )
       lowerCamelCase__				   =      decoders.Metaspace()

       if args.export_to_hub:
              logger.info("Exporting the trained tokenzier to Hub."      )
              lowerCamelCase__				   =      AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_      )
              new_tokenizer.push_to_hub("unigram-tokenizer-dataset"      )


if __name__ == "__main__":
    lowerCAmelCase_      =   parse_args()
    main(args)



import argparse

import torch

from transformers import ImageGPTConfig, ImageGPTForCausalLM, load_tf_weights_in_imagegpt
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()
def 					lowerCAmelCase__     (    SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_	)							->						List[str]:
			# Construct configuration depending on size
			lowerCAmelCase__ : Tuple   =						{'small': (512, 8, 24), 'medium': (1_024, 8, 36), 'large': (1_536, 16, 48)}
			lowerCAmelCase__ ,							lowerCAmelCase__ ,							lowerCAmelCase__ : Optional[int]   =						MODELS[model_size]  # set model hyperparameters
			lowerCAmelCase__ : Optional[int]   =						ImageGPTConfig(n_embd=SCREAMING_SNAKE_CASE_			,				n_layer=SCREAMING_SNAKE_CASE_			,				n_head=SCREAMING_SNAKE_CASE_	)
			lowerCAmelCase__ : List[Any]   =						ImageGPTForCausalLM(SCREAMING_SNAKE_CASE_	)

			# Load weights from numpy
			load_tf_weights_in_imagegpt(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_	)

			# Save pytorch-model
			lowerCAmelCase__ : Optional[int]   =						pytorch_dump_folder_path + '/' + WEIGHTS_NAME
			lowerCAmelCase__ : Optional[int]   =						pytorch_dump_folder_path + '/' + CONFIG_NAME
			print(F'''Save PyTorch model to {pytorch_weights_dump_path}'''	)
			torch.save(model.state_dict()			,				SCREAMING_SNAKE_CASE_	)
			print(F'''Save configuration file to {pytorch_config_dump_path}'''	)
			with open(SCREAMING_SNAKE_CASE_			,				'w'			,				encoding='utf-8'	) as f:
						f.write(config.to_json_string()	)


if __name__ == "__main__":
					lowerCamelCase__        =						argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--imagegpt_checkpoint_path""",
					    default=None,
					    type=str,
					    required=True,
					    help="""Path to the TensorFlow checkpoint path.""",
					)
					parser.add_argument(
					    """--model_size""",
					    default=None,
					    type=str,
					    required=True,
					    help="""Size of the model (can be either 'small', 'medium' or 'large').""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
					)
					lowerCamelCase__        =						parser.parse_args()
					convert_imagegpt_checkpoint_to_pytorch(
					    args.imagegpt_checkpoint_path, args.model_size, args.pytorch_dump_folder_path
					)