





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_UpperCamelCase       :				Optional[Any]           =			logging.getLogger(__name__)
def  __UpperCamelCase    (						)    ->		int:


   '''simple docstring'''







   __A    =  argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''      )
   parser.add_argument(
       '''--dataset_name'''							,			type=snake_case							,			default='''wikitext'''							,			help='''Name of the training. Explore datasets at: hf.co/datasets.'''							,			)
   parser.add_argument(
       '''--dataset_config'''							,			type=snake_case							,			default='''wikitext-103-raw-v1'''							,			help='''Configuration name of the dataset.'''      )
   parser.add_argument(
       '''--batch_size'''							,			type=snake_case							,			default=1_0_0_0							,			help='''Batch size during training.'''							,			)
   parser.add_argument(
       '''--vocab_size'''							,			type=snake_case							,			default=1_0_0_4_8							,			help='''Size of the desired vocabulary.'''							,			)
   parser.add_argument(
       '''--limit'''							,			default=snake_case							,			type=snake_case							,			help='''Limit the number of shards (used for debugging).'''							,			)
   parser.add_argument(
       '''--export_to_hub'''							,			action='''store_true'''							,			)

   __A    =  parser.parse_args()
   return args
def  __UpperCamelCase    (						snake_case      )    ->		Optional[int]:


   '''simple docstring'''







   __A    =  datasets.load_dataset(args.dataset_name							,			args.dataset_config							,			split='''train'''      )

   if args.limit is not None:
      __A    =  min(len(snake_case      )							,			args.limit      )
      __A    =  dataset.select(range(snake_case      )      )
      logger.info(F"Limiting the dataset to {args.limit} entries."      )

   def batch_iterator():
      for i in range(0							,			len(snake_case      )							,			args.batch_size      ):
         yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
   __A    =  Tokenizer(Unigram()      )
   __A    =  normalizers.Sequence([normalizers.Replace('''``'''							,			'''"'''      ), normalizers.Replace('''\'\''''							,			'''"'''      )]      )
   __A    =  pre_tokenizers.Metaspace()

   # Prepare the trainer.
   __A    =  UnigramTrainer(
       unk_token='''<unk>'''							,			special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']							,			vocab_size=args.vocab_size							,			)

   logger.info('''Training the tokenizer.'''      )
   tokenizer.train_from_iterator(batch_iterator()							,			trainer=snake_case      )
   logger.info('''Tokenizer training complete!'''      )

   __A    =  tokenizer.token_to_id('''[CLS]'''      )
   __A    =  tokenizer.token_to_id('''[SEP]'''      )
   __A    =  processors.TemplateProcessing(
       single='''[CLS]:0 $A:0 [SEP]:0'''							,			pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''							,			special_tokens=[
           ('''[CLS]''', cls_token_id),
           ('''[SEP]''', sep_token_id),
       ]							,			)
   __A    =  decoders.Metaspace()

   if args.export_to_hub:
      logger.info('''Exporting the trained tokenzier to Hub.'''      )
      __A    =  AlbertTokenizerFast(tokenizer_object=snake_case      )
      new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''      )


if __name__ == "__main__":
    _UpperCamelCase       :				Optional[Any]           =			parse_args()
    main(args)








class 		_lowerCAmelCase:

  """simple docstring"""







  def __init__(   self      )->   None:
     # numbermap keys are the number and its values are lists of indexes sorted
     # in ascending order
     __A    =  {}
     # indexmap keys are an index and it's values are the number at that index
     __A    =  {}

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   list[int]:
     if isinstance(UpperCAmelCase    ,							(range, str)      ):
        __A    =  list(UpperCAmelCase      )
     elif not isinstance(UpperCAmelCase    ,							UpperCAmelCase      ):
        raise TypeError(
            '''binary_search_delete() only accepts either a list, range or str'''      )

     __A    =  0
     __A    =  len(UpperCAmelCase      ) - 1

     while low <= high:
        __A    =  (low + high) // 2
        if array[mid] == item:
           array.pop(UpperCAmelCase      )
           return array
        elif array[mid] < item:
           __A    =  mid + 1
        else:
           __A    =  mid - 1
     raise ValueError(
         '''Either the item is not in the array or the array was unsorted'''      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   list[int]:
     if isinstance(UpperCAmelCase    ,							(range, str)      ):
        __A    =  list(UpperCAmelCase      )
     elif not isinstance(UpperCAmelCase    ,							UpperCAmelCase      ):
        raise TypeError(
            '''binary_search_insert() only accepts either a list, range or str'''      )

     __A    =  0
     __A    =  len(UpperCAmelCase      ) - 1

     while low <= high:
        __A    =  (low + high) // 2
        if array[mid] == index:
           # If the item already exists in the array,
           # insert it after the existing item
           array.insert(mid + 1    ,							UpperCAmelCase      )
           return array
        elif array[mid] < index:
           __A    =  mid + 1
        else:
           __A    =  mid - 1

        # If the item doesn't exist in the array, insert it at the appropriate position
     array.insert(UpperCAmelCase    ,							UpperCAmelCase      )
     return array

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   None:
     # Remove previous index
     if index in self.indexmap:
        __A    =  self.indexmap[index]
        if len(self.numbermap[n]      ) == 1:
           del self.numbermap[n]
        else:
           __A    =  self.binary_search_delete(self.numbermap[n]    ,							UpperCAmelCase      )

        # Set new index
     __A    =  number

     # Number not seen before or empty so insert number value
     if number not in self.numbermap:
        __A    =  [index]

     # Here we need to perform a binary search insertion in order to insert
     # The item in the correct place
     else:
        __A    =  self.binary_search_insert(
            self.numbermap[number]    ,							UpperCAmelCase      )





  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   int:
     return self.numbermap.get(UpperCAmelCase    ,							[-1]      )[0]


if __name__ == "__main__":
    import doctest

    doctest.testmod()

