


'''simple docstring'''







import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


a__    :   str								=			logging.getLogger(__name__)







def _UpperCamelCase			(	)				->						Dict:


      '''simple docstring'''



      UpperCamelCase__      =							argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."       )
      parser.add_argument(
          "--dataset_name" ,     type=__A ,     default="wikitext" ,     help="Name of the training. Explore datasets at: hf.co/datasets." ,     )
      parser.add_argument(
          "--dataset_config" ,     type=__A ,     default="wikitext-103-raw-v1" ,     help="Configuration name of the dataset."       )
      parser.add_argument(
          "--batch_size" ,     type=__A ,     default=1000 ,     help="Batch size during training." ,     )
      parser.add_argument(
          "--vocab_size" ,     type=__A ,     default=10048 ,     help="Size of the desired vocabulary." ,     )
      parser.add_argument(
          "--limit" ,     default=__A ,     type=__A ,     help="Limit the number of shards (used for debugging)." ,     )
      parser.add_argument(
          "--export_to_hub" ,     action="store_true" ,     )

      UpperCamelCase__      =							parser.parse_args()
      return args







def _UpperCamelCase			(	__A       )				->						Optional[int]:


      '''simple docstring'''



      UpperCamelCase__      =							datasets.load_dataset(args.dataset_name ,     args.dataset_config ,     split="train"       )

      if args.limit is not None:
            UpperCamelCase__      =							min(len(__A       ) ,     args.limit       )
            UpperCamelCase__      =							dataset.select(range(__A       )       )
            logger.info(F'''Limiting the dataset to {args.limit} entries.'''       )

      def batch_iterator():
            for i in range(0 ,     len(__A       ) ,     args.batch_size       ):
                  yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
      UpperCamelCase__      =							Tokenizer(Unigram()       )
      UpperCamelCase__      =							normalizers.Sequence([normalizers.Replace("``" ,     "\""       ), normalizers.Replace("''" ,     "\""       )]       )
      UpperCamelCase__      =							pre_tokenizers.Metaspace()

      # Prepare the trainer.
      UpperCamelCase__      =							UnigramTrainer(
          unk_token="<unk>" ,     special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"] ,     vocab_size=args.vocab_size ,     )

      logger.info("Training the tokenizer."       )
      tokenizer.train_from_iterator(batch_iterator() ,     trainer=__A       )
      logger.info("Tokenizer training complete!"       )

      UpperCamelCase__      =							tokenizer.token_to_id("[CLS]"       )
      UpperCamelCase__      =							tokenizer.token_to_id("[SEP]"       )
      UpperCamelCase__      =							processors.TemplateProcessing(
          single="[CLS]:0 $A:0 [SEP]:0" ,     pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1" ,     special_tokens=[
              ("[CLS]", cls_token_id),
              ("[SEP]", sep_token_id),
          ] ,     )
      UpperCamelCase__      =							decoders.Metaspace()

      if args.export_to_hub:
            logger.info("Exporting the trained tokenzier to Hub."       )
            UpperCamelCase__      =							AlbertTokenizerFast(tokenizer_object=__A       )
            new_tokenizer.push_to_hub("unigram-tokenizer-dataset"       )


if __name__ == "__main__":
  a__    :   Any								=			parse_args()
  main(args)









'''simple docstring'''







import itertools







def _UpperCamelCase			(	__A = 1000000       )				->						int:


      '''simple docstring'''



      UpperCamelCase__      =							[1]

      for i in itertools.count(len(__A       )       ):
            UpperCamelCase__      =							0
            for j in itertools.count(1       ):
                  UpperCamelCase__      =							-1 if j % 2 == 0 else +1
                  UpperCamelCase__      =							(j * j * 3 - j) // 2
                  if index > i:
                        break
                  item += partitions[i - index] * sign
                  item %= number
                  index += j
                  if index > i:
                        break
                  item += partitions[i - index] * sign
                  item %= number

            if item == 0:
                  return i
            partitions.append(__A       )

      return 0


if __name__ == "__main__":
  import doctest

  doctest.testmod()

  print(F"""{solution() = }""")





