





"""simple docstring"""





import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter





class 							_lowerCamelCase   (							TensorFormatter[Mapping, np.ndarray, Mapping]			):


						def __init__(self ,				__a=None ,				**__a		)			-> int:
								super().__init__(features=__a		)
								UpperCamelCase =						np_array_kwargs


						def      snake_case_ (self ,				__a		)			-> int:
								if isinstance(__a ,				__a		):
										if column and all(
										    isinstance(__a ,				np.ndarray		) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column		):
												return np.stack(__a		)
										else:
												# don't use np.array(column, dtype=object)
												# since it fails in certain cases
												# see https://stackoverflow.com/q/51005699
												UpperCamelCase =						np.empty(len(__a		) ,				dtype=__a		)
												UpperCamelCase =						column
												return out
								return column


						def      snake_case_ (self ,				__a		)			-> Dict:
								if isinstance(__a ,				(str, bytes, type(__a		))		):
										return value
								elif isinstance(__a ,				(np.character, np.ndarray)		) and np.issubdtype(value.dtype ,				np.character		):
										return value
								elif isinstance(__a ,				np.number		):
										return value

								UpperCamelCase =						{}

								if isinstance(__a ,				np.ndarray		) and np.issubdtype(value.dtype ,				np.integer		):
										UpperCamelCase =						{"dtype": np.intaa}
								elif isinstance(__a ,				np.ndarray		) and np.issubdtype(value.dtype ,				np.floating		):
										UpperCamelCase =						{"dtype": np.floataa}
								elif config.PIL_AVAILABLE and "PIL" in sys.modules:
										import PIL.Image

										if isinstance(__a ,				PIL.Image.Image		):
												return np.asarray(__a ,				**self.np_array_kwargs		)

								return np.asarray(__a ,				**{**default_dtype, **self.np_array_kwargs}		)


						def      snake_case_ (self ,				__a		)			-> List[Any]:
								# support for torch, tf, jax etc.
								if config.TORCH_AVAILABLE and "torch" in sys.modules:
										import torch

										if isinstance(__a ,				torch.Tensor		):
												return self._tensorize(data_struct.detach().cpu().numpy()[()]		)
								if hasattr(__a ,				"__array__"		) and not isinstance(__a ,				(np.ndarray, np.character, np.number)		):
										UpperCamelCase =						data_struct.__array__()
								# support for nested types like struct of list of struct
								if isinstance(__a ,				np.ndarray		):
										if data_struct.dtype == object:
												return self._consolidate([self.recursive_tensorize(__a		) for substruct in data_struct]		)
								if isinstance(__a ,				(list, tuple)		):
										return self._consolidate([self.recursive_tensorize(__a		) for substruct in data_struct]		)
								return self._tensorize(__a		)


						def      snake_case_ (self ,				__a		)			-> Tuple:
								return map_nested(self._recursive_tensorize ,				__a ,				map_list=__a		)


						def      snake_case_ (self ,				__a		)			-> Mapping:
								UpperCamelCase =						self.numpy_arrow_extractor().extract_row(__a		)
								UpperCamelCase =						self.python_features_decoder.decode_row(__a		)
								return self.recursive_tensorize(__a		)


						def      snake_case_ (self ,				__a		)			-> np.ndarray:
								UpperCamelCase =						self.numpy_arrow_extractor().extract_column(__a		)
								UpperCamelCase =						self.python_features_decoder.decode_column(__a ,				pa_table.column_names[0]		)
								UpperCamelCase =						self.recursive_tensorize(__a		)
								UpperCamelCase =						self._consolidate(__a		)
								return column




						def      snake_case_ (self ,				__a		)			-> Mapping:
								UpperCamelCase =						self.numpy_arrow_extractor().extract_batch(__a		)
								UpperCamelCase =						self.python_features_decoder.decode_batch(__a		)
								UpperCamelCase =						self.recursive_tensorize(__a		)
								for column_name in batch:
										UpperCamelCase =						self._consolidate(batch[column_name]		)
								return batch













"""simple docstring"""





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowerCAmelCase__        =      logging.getLogger(__name__)


def    a__       (       ):




		"""simple docstring"""


		UpperCamelCase =						argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."     )
		parser.add_argument(
		    "--dataset_name"	,						type=_SCREAMING_SNAKE_CASE	,						default="wikitext"	,						help="Name of the training. Explore datasets at: hf.co/datasets."	,						)
		parser.add_argument(
		    "--dataset_config"	,						type=_SCREAMING_SNAKE_CASE	,						default="wikitext-103-raw-v1"	,						help="Configuration name of the dataset."     )
		parser.add_argument(
		    "--batch_size"	,						type=_SCREAMING_SNAKE_CASE	,						default=1_000	,						help="Batch size during training."	,						)
		parser.add_argument(
		    "--vocab_size"	,						type=_SCREAMING_SNAKE_CASE	,						default=10_048	,						help="Size of the desired vocabulary."	,						)
		parser.add_argument(
		    "--limit"	,						default=_SCREAMING_SNAKE_CASE	,						type=_SCREAMING_SNAKE_CASE	,						help="Limit the number of shards (used for debugging)."	,						)
		parser.add_argument(
		    "--export_to_hub"	,						action="store_true"	,						)

		UpperCamelCase =						parser.parse_args()
		return args


def    a__       (       _SCREAMING_SNAKE_CASE     ):




		"""simple docstring"""


		UpperCamelCase =						datasets.load_dataset(args.dataset_name	,						args.dataset_config	,						split="train"     )

		if args.limit is not None:
				UpperCamelCase =						min(len(_SCREAMING_SNAKE_CASE     )	,						args.limit     )
				UpperCamelCase =						dataset.select(range(_SCREAMING_SNAKE_CASE     )     )
				logger.info(F"Limiting the dataset to {args.limit} entries."     )

		def batch_iterator():
				for i in range(0	,						len(_SCREAMING_SNAKE_CASE     )	,						args.batch_size     ):
						yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
		UpperCamelCase =						Tokenizer(Unigram()     )
		UpperCamelCase =						normalizers.Sequence([normalizers.Replace("``"	,						"\""     ), normalizers.Replace("''"	,						"\""     )]     )
		UpperCamelCase =						pre_tokenizers.Metaspace()

		# Prepare the trainer.
		UpperCamelCase =						UnigramTrainer(
		    unk_token="<unk>"	,						special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]	,						vocab_size=args.vocab_size	,						)

		logger.info("Training the tokenizer."     )
		tokenizer.train_from_iterator(batch_iterator()	,						trainer=_SCREAMING_SNAKE_CASE     )
		logger.info("Tokenizer training complete!"     )

		UpperCamelCase =						tokenizer.token_to_id("[CLS]"     )
		UpperCamelCase =						tokenizer.token_to_id("[SEP]"     )
		UpperCamelCase =						processors.TemplateProcessing(
		    single="[CLS]:0 $A:0 [SEP]:0"	,						pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"	,						special_tokens=[
		        ("[CLS]", cls_token_id),
		        ("[SEP]", sep_token_id),
		    ]	,						)
		UpperCamelCase =						decoders.Metaspace()

		if args.export_to_hub:
				logger.info("Exporting the trained tokenzier to Hub."     )
				UpperCamelCase =						AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE     )
				new_tokenizer.push_to_hub("unigram-tokenizer-dataset"     )


if __name__ == "__main__":
						lowerCAmelCase__        =      parse_args()
						main(args)






