



'''simple docstring'''







import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_UpperCamelCase  						=			logging.getLogger(__name__)
def 			_lowerCAmelCase(      ) ->							Union[str, Any]:
							lowerCAmelCase__  =   argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""						)
							parser.add_argument(
							    """--dataset_name"""					, type=UpperCAmelCase_					, default="""wikitext"""					, help="""Name of the training. Explore datasets at: hf.co/datasets."""					, )
							parser.add_argument(
							    """--dataset_config"""					, type=UpperCAmelCase_					, default="""wikitext-103-raw-v1"""					, help="""Configuration name of the dataset."""						)
							parser.add_argument(
							    """--batch_size"""					, type=UpperCAmelCase_					, default=1000					, help="""Batch size during training."""					, )
							parser.add_argument(
							    """--vocab_size"""					, type=UpperCAmelCase_					, default=10048					, help="""Size of the desired vocabulary."""					, )
							parser.add_argument(
							    """--limit"""					, default=UpperCAmelCase_					, type=UpperCAmelCase_					, help="""Limit the number of shards (used for debugging)."""					, )
							parser.add_argument(
							    """--export_to_hub"""					, action="""store_true"""					, )

							lowerCAmelCase__  =   parser.parse_args()
							return args
def 			_lowerCAmelCase(      UpperCAmelCase_      :	List[Any]						) ->							str:
							lowerCAmelCase__  =   datasets.load_dataset(args.dataset_name					, args.dataset_config					, split="""train"""						)

							if args.limit is not None:
														lowerCAmelCase__  =   min(len(UpperCAmelCase_						)					, args.limit						)
														lowerCAmelCase__  =   dataset.select(range(UpperCAmelCase_						)						)
														logger.info(F'''Limiting the dataset to {args.limit} entries.'''						)

							def batch_iterator():
														for i in range(0					, len(UpperCAmelCase_						)					, args.batch_size						):
																					yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
							lowerCAmelCase__  =   Tokenizer(Unigram()						)
							lowerCAmelCase__  =   normalizers.Sequence([normalizers.Replace("""``"""					, """\""""						), normalizers.Replace("""''"""					, """\""""						)]						)
							lowerCAmelCase__  =   pre_tokenizers.Metaspace()

							# Prepare the trainer.
							lowerCAmelCase__  =   UnigramTrainer(
							    unk_token="""<unk>"""					, special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]					, vocab_size=args.vocab_size					, )

							logger.info("""Training the tokenizer."""						)
							tokenizer.train_from_iterator(batch_iterator()					, trainer=UpperCAmelCase_						)
							logger.info("""Tokenizer training complete!"""						)

							lowerCAmelCase__  =   tokenizer.token_to_id("""[CLS]"""						)
							lowerCAmelCase__  =   tokenizer.token_to_id("""[SEP]"""						)
							lowerCAmelCase__  =   processors.TemplateProcessing(
							    single="""[CLS]:0 $A:0 [SEP]:0"""					, pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""					, special_tokens=[
							        ("""[CLS]""", cls_token_id),
							        ("""[SEP]""", sep_token_id),
							    ]					, )
							lowerCAmelCase__  =   decoders.Metaspace()

							if args.export_to_hub:
														logger.info("""Exporting the trained tokenzier to Hub."""						)
														lowerCAmelCase__  =   AlbertTokenizerFast(tokenizer_object=UpperCAmelCase_						)
														new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""						)


if __name__ == "__main__":
		_UpperCamelCase  						=			parse_args()
		main(args)










'''simple docstring'''







from ..utils import DummyObject, requires_backends

class      lowerCamelCase__     (     metaclass=_A				):






	'''simple docstring'''

	A__		=	['''torch''', '''librosa''']




	def __init__(  self		:  Optional[Any]   ,		*__A		:  Tuple   ,		**__A		:  Tuple					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(self   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Dict   ,		*__A		:  List[Any]   ,		**__A		:  List[str]					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Optional[int]   ,		*__A		:  List[str]   ,		**__A		:  Tuple					)     ->			Any:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)





class      lowerCamelCase__     (     metaclass=_A				):






	'''simple docstring'''

	A__		=	['''torch''', '''librosa''']




	def __init__(  self		:  int   ,		*__A		:  Dict   ,		**__A		:  List[Any]					)     ->			List[Any]:




								'''simple docstring'''





								requires_backends(self   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Optional[int]   ,		*__A		:  Optional[int]   ,		**__A		:  List[Any]					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  List[str]   ,		*__A		:  List[str]   ,		**__A		:  int					)     ->			List[str]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)





