



"""simple docstring"""


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


snake_case_								=			logging.getLogger(__name__)

def 					_lowerCAmelCase (  ):
						UpperCAmelCase			 =					argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'		)
						parser.add_argument(
						    '--dataset_name'						,      type=UpperCAmelCase_						,      default='wikitext'						,      help='Name of the training. Explore datasets at: hf.co/datasets.'						,      )
						parser.add_argument(
						    '--dataset_config'						,      type=UpperCAmelCase_						,      default='wikitext-103-raw-v1'						,      help='Configuration name of the dataset.'		)
						parser.add_argument(
						    '--batch_size'						,      type=UpperCAmelCase_						,      default=1000						,      help='Batch size during training.'						,      )
						parser.add_argument(
						    '--vocab_size'						,      type=UpperCAmelCase_						,      default=10048						,      help='Size of the desired vocabulary.'						,      )
						parser.add_argument(
						    '--limit'						,      default=UpperCAmelCase_						,      type=UpperCAmelCase_						,      help='Limit the number of shards (used for debugging).'						,      )
						parser.add_argument(
						    '--export_to_hub'						,      action='store_true'						,      )

						UpperCAmelCase			 =					parser.parse_args()
						return args

def 					_lowerCAmelCase (  lowercase_		):
						UpperCAmelCase			 =					datasets.load_dataset(args.dataset_name						,      args.dataset_config						,      split='train'		)

						if args.limit is not None:
												UpperCAmelCase			 =					min(len(UpperCAmelCase_		)						,      args.limit		)
												UpperCAmelCase			 =					dataset.select(range(UpperCAmelCase_		)		)
												logger.info(F"""Limiting the dataset to {args.limit} entries."""		)

						def batch_iterator():
												for i in range(0						,      len(UpperCAmelCase_		)						,      args.batch_size		):
																		yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
						UpperCAmelCase			 =					Tokenizer(Unigram()		)
						UpperCAmelCase			 =					normalizers.Sequence([normalizers.Replace('``'						,      '\"'		), normalizers.Replace('\'\''						,      '\"'		)]		)
						UpperCAmelCase			 =					pre_tokenizers.Metaspace()

						# Prepare the trainer.
						UpperCAmelCase			 =					UnigramTrainer(
						    unk_token='<unk>'						,      special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']						,      vocab_size=args.vocab_size						,      )

						logger.info('Training the tokenizer.'		)
						tokenizer.train_from_iterator(batch_iterator()						,      trainer=UpperCAmelCase_		)
						logger.info('Tokenizer training complete!'		)

						UpperCAmelCase			 =					tokenizer.token_to_id('[CLS]'		)
						UpperCAmelCase			 =					tokenizer.token_to_id('[SEP]'		)
						UpperCAmelCase			 =					processors.TemplateProcessing(
						    single='[CLS]:0 $A:0 [SEP]:0'						,      pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'						,      special_tokens=[
						        ('[CLS]', cls_token_id),
						        ('[SEP]', sep_token_id),
						    ]						,      )
						UpperCAmelCase			 =					decoders.Metaspace()

						if args.export_to_hub:
												logger.info('Exporting the trained tokenzier to Hub.'		)
												UpperCAmelCase			 =					AlbertTokenizerFast(tokenizer_object=UpperCAmelCase_		)
												new_tokenizer.push_to_hub('unigram-tokenizer-dataset'		)


if __name__ == "__main__":
				snake_case_								=			parse_args()
				main(args)










'''simple docstring'''







def 			_lowerCAmelCase(      UpperCAmelCase_      :	list					, UpperCAmelCase_      :	int					, UpperCAmelCase_      :	int					, UpperCAmelCase_      :	int						) ->							int:
							lowerCAmelCase__  =   int((lower_bound + upper_bound) // 2						)
							if array[r] == value:
														return r
							if lower_bound >= upper_bound:
														return -1
							if array[r] < value:
														return binary_search(UpperCAmelCase_					, r + 1					, UpperCAmelCase_					, UpperCAmelCase_						)
							else:
														return binary_search(UpperCAmelCase_					, UpperCAmelCase_					, r - 1					, UpperCAmelCase_						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	int					, UpperCAmelCase_      :	list						) ->							list:
							lowerCAmelCase__  =   0
							if matrix[index][0] == value:
														return [index, 0]
							while index < len(UpperCAmelCase_						) and matrix[index][0] < value:
														lowerCAmelCase__  =   binary_search(matrix[index]					, 0					, len(matrix[index]						) - 1					, UpperCAmelCase_						)
														if r != -1:
																					return [index, r]
														index += 1
							return [-1, -1]


if __name__ == "__main__":
		import doctest

		doctest.testmod()





