






import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCamelCase												=      logging.getLogger(__name__)

def 			__lowerCamelCase     (						)				->    Tuple:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE       =    argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""						)
						parser.add_argument(
						    """--dataset_name"""			,type=snake_case__			,default="""wikitext"""			,help="""Name of the training. Explore datasets at: hf.co/datasets."""			,)
						parser.add_argument(
						    """--dataset_config"""			,type=snake_case__			,default="""wikitext-103-raw-v1"""			,help="""Configuration name of the dataset."""						)
						parser.add_argument(
						    """--batch_size"""			,type=snake_case__			,default=10_00			,help="""Batch size during training."""			,)
						parser.add_argument(
						    """--vocab_size"""			,type=snake_case__			,default=1_00_48			,help="""Size of the desired vocabulary."""			,)
						parser.add_argument(
						    """--limit"""			,default=snake_case__			,type=snake_case__			,help="""Limit the number of shards (used for debugging)."""			,)
						parser.add_argument(
						    """--export_to_hub"""			,action="""store_true"""			,)

						_SCREAMING_SNAKE_CASE       =    parser.parse_args()
						return args

def 			__lowerCamelCase     (						snake_case__						)				->    Optional[int]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE       =    datasets.load_dataset(args.dataset_name			,args.dataset_config			,split="""train"""						)

						if args.limit is not None:
												_SCREAMING_SNAKE_CASE       =    min(len(snake_case__						)			,args.limit						)
												_SCREAMING_SNAKE_CASE       =    dataset.select(range(snake_case__						)						)
												logger.info(F'Limiting the dataset to {args.limit} entries.'						)

						def batch_iterator():
												for i in range(0			,len(snake_case__						)			,args.batch_size						):
																		yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
						_SCREAMING_SNAKE_CASE       =    Tokenizer(Unigram()						)
						_SCREAMING_SNAKE_CASE       =    normalizers.Sequence([normalizers.Replace("""``"""			,"""\""""						), normalizers.Replace("""''"""			,"""\""""						)]						)
						_SCREAMING_SNAKE_CASE       =    pre_tokenizers.Metaspace()

						# Prepare the trainer.
						_SCREAMING_SNAKE_CASE       =    UnigramTrainer(
						    unk_token="""<unk>"""			,special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]			,vocab_size=args.vocab_size			,)

						logger.info("""Training the tokenizer."""						)
						tokenizer.train_from_iterator(batch_iterator()			,trainer=snake_case__						)
						logger.info("""Tokenizer training complete!"""						)

						_SCREAMING_SNAKE_CASE       =    tokenizer.token_to_id("""[CLS]"""						)
						_SCREAMING_SNAKE_CASE       =    tokenizer.token_to_id("""[SEP]"""						)
						_SCREAMING_SNAKE_CASE       =    processors.TemplateProcessing(
						    single="""[CLS]:0 $A:0 [SEP]:0"""			,pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""			,special_tokens=[
						        ("""[CLS]""", cls_token_id),
						        ("""[SEP]""", sep_token_id),
						    ]			,)
						_SCREAMING_SNAKE_CASE       =    decoders.Metaspace()

						if args.export_to_hub:
												logger.info("""Exporting the trained tokenzier to Hub."""						)
												_SCREAMING_SNAKE_CASE       =    AlbertTokenizerFast(tokenizer_object=snake_case__						)
												new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""						)


if __name__ == "__main__":
		UpperCamelCase												=      parse_args()
		main(args)









from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
		if not (is_transformers_available() and is_torch_available() and is_transformers_version('''>=''', '''4.27.0''')):
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		from ...utils.dummy_torch_and_transformers_objects import (
		    AudioLDMPipeline,
		)
else:
		from .pipeline_audioldm import AudioLDMPipeline

