





from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


UpperCamelCase__ :			int       	=			logging.get_logger(__name__)

UpperCamelCase__ :			Any       	=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase__ :			Any       	=			{
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase__ :			Dict       	=			{
    """openai/clip-vit-base-patch32""": 77,
}







class 	_UpperCamelCase	(							snake_case_	):







					'''simple docstring'''

					lowerCamelCase  :  Any           =							VOCAB_FILES_NAMES
					lowerCamelCase  :  str           =							PRETRAINED_VOCAB_FILES_MAP
					lowerCamelCase  :  List[str]           =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCamelCase  :  Optional[Any]           =							['input_ids', 'attention_mask']
					lowerCamelCase  :  List[str]           =							CLIPTokenizer





					def __init__(    self     :	Dict			,       __lowercase     :	Optional[int]=None			,       __lowercase     :	Tuple=None			,       __lowercase     :	int=None			,       __lowercase     :	Any="<|endoftext|>"			,       __lowercase     :	List[str]="<|startoftext|>"			,       __lowercase     :	List[str]="<|endoftext|>"			,       __lowercase     :	int="<|endoftext|>"			,       **__lowercase     :	int			,       ):






						'''simple docstring'''







						super().__init__(
						    lowercase_			,       lowercase_			,       tokenizer_file=lowercase_			,       unk_token=lowercase_			,       bos_token=lowercase_			,       eos_token=lowercase_			,       pad_token=lowercase_			,       **lowercase_			,       )

						if not isinstance(self.backend_tokenizer.pre_tokenizer			,       pre_tokenizers.Sequence		):
							raise ValueError(
							    """The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"""
							    """ heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"""
							    """ to be compatible with this version.The easiest way to do so is"""
							    """ `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"""
							    """ to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"""
							    """ transformers."""		)

						self._wrap_decode_method_backend_tokenizer()





					def 							SCREAMING_SNAKE_CASE				(    self     :	Any		):






						'''simple docstring'''







						UpperCAmelCase_		    =  self.backend_tokenizer.decode

						def new_decode_method(*__lowercase     :	Optional[Any]			,       **__lowercase     :	List[Any]		):
							UpperCAmelCase_		    =  orig_decode_method(*lowercase_			,       **lowercase_		)
							UpperCAmelCase_		    =  text.replace(self.backend_tokenizer.model.end_of_word_suffix			,       """ """		).strip()
							return text

						UpperCAmelCase_		    =  new_decode_method





					def 							SCREAMING_SNAKE_CASE				(    self     :	List[str]			,       __lowercase     :	List[int]			,       __lowercase     :	Optional[List[int]] = None		):






						'''simple docstring'''







						UpperCAmelCase_		    =  [self.bos_token_id]
						UpperCAmelCase_		    =  [self.eos_token_id]

						if token_ids_a is None:
							return bos_token + token_ids_a + eos_token
						return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token





					def 							SCREAMING_SNAKE_CASE				(    self     :	Tuple			,       __lowercase     :	List[int]			,       __lowercase     :	Optional[List[int]] = None		):






						'''simple docstring'''







						UpperCAmelCase_		    =  [self.bos_token_id]
						UpperCAmelCase_		    =  [self.eos_token_id]

						if token_ids_a is None:
							return len(bos_token + token_ids_a + eos_token		) * [0]
						return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token		) * [0]





					def 							SCREAMING_SNAKE_CASE				(    self     :	Tuple			,       __lowercase     :	str			,       __lowercase     :	Optional[str] = None		):






						'''simple docstring'''







						UpperCAmelCase_		    =  self._tokenizer.model.save(lowercase_			,       name=lowercase_		)
						return tuple(lowercase_		)










"""simple docstring"""


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_UpperCamelCase :    List[str]      =			logging.getLogger(__name__)




def 					a			(  )	->		Union[str, Any]:
 __A						:							int      =						argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'		)
 parser.add_argument(
     '--dataset_name'  ,						type=UpperCamelCase_  ,						default='wikitext'  ,						help='Name of the training. Explore datasets at: hf.co/datasets.'  ,						)
 parser.add_argument(
     '--dataset_config'  ,						type=UpperCamelCase_  ,						default='wikitext-103-raw-v1'  ,						help='Configuration name of the dataset.'		)
 parser.add_argument(
     '--batch_size'  ,						type=UpperCamelCase_  ,						default=1_0_0_0  ,						help='Batch size during training.'  ,						)
 parser.add_argument(
     '--vocab_size'  ,						type=UpperCamelCase_  ,						default=1_0_0_4_8  ,						help='Size of the desired vocabulary.'  ,						)
 parser.add_argument(
     '--limit'  ,						default=UpperCamelCase_  ,						type=UpperCamelCase_  ,						help='Limit the number of shards (used for debugging).'  ,						)
 parser.add_argument(
     '--export_to_hub'  ,						action='store_true'  ,						)

 __A						:							List[Any]      =						parser.parse_args()
 return args




def 					a			(  UpperCamelCase_				:					Optional[Any]		)	->		Optional[Any]:
 __A						:							Union[str, Any]      =						datasets.load_dataset(args.dataset_name  ,						args.dataset_config  ,						split='train'		)

 if args.limit is not None:
  __A						:							Tuple      =						min(len(UpperCamelCase_		)  ,						args.limit		)
  __A						:							Optional[int]      =						dataset.select(range(UpperCamelCase_		)		)
  logger.info(F"Limiting the dataset to {args.limit} entries."		)

 def batch_iterator():
  for i in range(0  ,						len(UpperCamelCase_		)  ,						args.batch_size		):
   yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
 __A						:							Any      =						Tokenizer(Unigram()		)
 __A						:							List[Any]      =						normalizers.Sequence([normalizers.Replace('``'  ,						'"'		), normalizers.Replace('\'\''  ,						'"'		)]		)
 __A						:							Optional[int]      =						pre_tokenizers.Metaspace()

 # Prepare the trainer.
 __A						:							int      =						UnigramTrainer(
     unk_token='<unk>'  ,						special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']  ,						vocab_size=args.vocab_size  ,						)

 logger.info('Training the tokenizer.'		)
 tokenizer.train_from_iterator(batch_iterator()  ,						trainer=UpperCamelCase_		)
 logger.info('Tokenizer training complete!'		)

 __A						:							Union[str, Any]      =						tokenizer.token_to_id('[CLS]'		)
 __A						:							str      =						tokenizer.token_to_id('[SEP]'		)
 __A						:							Tuple      =						processors.TemplateProcessing(
     single='[CLS]:0 $A:0 [SEP]:0'  ,						pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'  ,						special_tokens=[
         ('[CLS]', cls_token_id),
         ('[SEP]', sep_token_id),
     ]  ,						)
 __A						:							List[str]      =						decoders.Metaspace()

 if args.export_to_hub:
  logger.info('Exporting the trained tokenzier to Hub.'		)
  __A						:							Union[str, Any]      =						AlbertTokenizerFast(tokenizer_object=UpperCamelCase_		)
  new_tokenizer.push_to_hub('unigram-tokenizer-dataset'		)


if __name__ == "__main__":
   _UpperCamelCase :    Optional[Any]      =			parse_args()
   main(args)





