
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase	      = logging.getLogger(__name__)






def 							UpperCAmelCase_		(							):
	lowercase       =    argparse.ArgumentParser(
	    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'							)
	parser.add_argument(
	    '--dataset_name' ,							type=_snake_case ,							default='wikitext' ,							help='Name of the training. Explore datasets at: hf.co/datasets.' ,							)
	parser.add_argument(
	    '--dataset_config' ,							type=_snake_case ,							default='wikitext-103-raw-v1' ,							help='Configuration name of the dataset.'							)
	parser.add_argument(
	    '--tokenizer_name_or_path' ,							type=_snake_case ,							default='sayakpaul/unigram-tokenizer-wikitext' ,							help='Tokenizer identifier. Can be a local filepath or a Hub identifier.' ,							)
	parser.add_argument(
	    '--shard_size' ,							type=_snake_case ,							default=1000 ,							help='Number of entries to go in a single shard.' ,							)
	parser.add_argument('--split' ,							type=_snake_case ,							default='train' ,							choices=['train', 'test', 'validation']							)
	parser.add_argument(
	    '--limit' ,							default=_snake_case ,							type=_snake_case ,							help='Limit the number of shards (used for debugging).' ,							)
	parser.add_argument(
	    '--max_length' ,							type=_snake_case ,							default=512 ,							help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
	    ' sequence length that is a multiple of 8.' ,							)
	parser.add_argument(
	    '--output_dir' ,							default='tf-tpu' ,							type=_snake_case ,							help='Output directory where the TFRecord shards will be saved. If the'
	    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
	    ' shards will be directly saved to a Google Cloud Storage bucket.' ,							)

	lowercase       =    parser.parse_args()
	return args






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	def fn(__SCREAMING_SNAKE_CASE							):
		return tokenizer(examples['text']							)

	return fn






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    []
	for i in range(len(tokenized_data['input_ids']							)							):
		lowercase       =    {
		    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]							)							),
		    '''attention_mask''': tf.train.Feature(
		        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]							)							),
		}
		lowercase       =    tf.train.Features(feature=_snake_case							)
		lowercase       =    tf.train.Example(features=_snake_case							)
		lowercase       =    example.SerializeToString()
		records.append(_snake_case							)
	return records






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    datasets.load_dataset(args.dataset_name ,							args.dataset_config ,							split=args.split							)

	if args.limit is not None:
		lowercase       =    min(len(_snake_case							) ,							args.limit							)
		lowercase       =    dataset.select(range(_snake_case							)							)
		print(F'''Limiting the dataset to {args.limit} entries.'''							)

	lowercase       =    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

	# Handle output directory creation.
	# For serializing into a Google Cloud Storage Bucket, one needs to first
	# create a bucket.
	if "gs" not in args.output_dir:
		if not os.path.exists(args.output_dir							):
			os.makedirs(args.output_dir							)
		lowercase       =    os.path.join(args.output_dir ,							args.split							)
		if not os.path.exists(_snake_case							):
			os.makedirs(_snake_case							)
	else:
		lowercase       =    os.path.join(args.output_dir ,							args.split							)

	# Tokenize the whole dataset at once.
	lowercase       =    tokenize_function(_snake_case							)
	lowercase       =    dataset.map(_snake_case ,							batched=_snake_case ,							num_proc=4 ,							remove_columns=['text']							)

	# We need to concatenate all our texts together, and then split the result
	# into chunks of a fixed size, which we will call block_size. To do this, we
	# will use the map method again, with the option batched=True. When we use batched=True,
	# the function we pass to map() will be passed multiple inputs at once, allowing us
	# to group them into more or fewer examples than we had in the input.
	# This allows us to create our new fixed-length samples. The advantage of this
	# method is that we don't lose a whole lot of content from the dataset compared to the
	# case where we simply tokenize with a pre-defined max_length.

	def group_texts(__SCREAMING_SNAKE_CASE							):
		# Concatenate all texts.
		lowercase       =    {k: sum(examples[k] ,							[]							) for k in examples.keys()}
		lowercase       =    len(concatenated_examples[list(examples.keys()							)[0]]							)
		# We drop the small remainder, though you could add padding instead if the model supports it
		# In this, as in all things, we advise you to follow your heart 🫀
		lowercase       =    (total_length // args.max_length) * args.max_length
		# Split by chunks of max_len.
		lowercase       =    {
		    k: [t[i : i + args.max_length] for i in range(0 ,							_snake_case ,							args.max_length							)]
		    for k, t in concatenated_examples.items()
		}
		return result

	lowercase       =    dataset_tokenized.map(_snake_case ,							batched=_snake_case ,							batch_size=1000 ,							num_proc=4							)

	lowercase       =    0
	lowercase       =    0
	for shard in range(0 ,							len(_snake_case							) ,							args.shard_size							):
		lowercase       =    grouped_dataset[shard : shard + args.shard_size]
		lowercase       =    len(dataset_snapshot['input_ids']							)
		lowercase       =    os.path.join(_snake_case ,							F'''dataset-{shard_count}-{records_containing}.tfrecord'''							)
		lowercase       =    get_serialized_examples(_snake_case							)

		with tf.io.TFRecordWriter(_snake_case							) as out_file:
			for i in range(len(_snake_case							)							):
				lowercase       =    serialized_examples[i]
				out_file.write(_snake_case							)
			print('Wrote file {} containing {} records'.format(_snake_case ,							_snake_case							)							)

		shard_count += 1
		total_records += records_containing

	with open(F'''split-{args.split}-records-count.txt''' ,							'w'							) as f:
		print(F'''Total {args.split} records: {total_records}''' ,							file=_snake_case							)


if __name__ == "__main__":
	UpperCAmelCase	      = parse_args()
	main(args)





"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_fnet import FNetTokenizer
else:
 snake_case__						:   str			 = None

snake_case__						:   Optional[Any]			 = logging.get_logger(__name__)
snake_case__						:   Optional[int]			 = {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case__						:   Dict			 = {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json''',
    },
}

snake_case__						:   Any			 = {
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

snake_case__						:   Dict			 = '''▁'''



class 						snake_case_(				a__	):
     __UpperCamelCase     =    VOCAB_FILES_NAMES
     __UpperCamelCase     =    PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase     =    ['''input_ids''', '''token_type_ids''']
     __UpperCamelCase     =    FNetTokenizer






     def __init__(		self     :    Union[str, Any]    , UpperCamelCase_     :    Union[str, Any]=None    , UpperCamelCase_     :    Union[str, Any]=None    , UpperCamelCase_     :    Any=False    , UpperCamelCase_     :    Any=True    , UpperCamelCase_     :    Dict=True    , UpperCamelCase_     :    Tuple="<unk>"    , UpperCamelCase_     :    List[str]="[SEP]"    , UpperCamelCase_     :    List[Any]="<pad>"    , UpperCamelCase_     :    Union[str, Any]="[CLS]"    , UpperCamelCase_     :    int="[MASK]"    , **UpperCamelCase_     :    Optional[Any]    , ):
           # Mask token behave like a normal word, i.e. include the space before it and
           # is included in the raw text, there should be a match in a non-normalized sentence.
           lowerCAmelCase    :		int				=      (
               AddedToken(UpperCamelCase_    , lstrip=UpperCamelCase_    , rstrip=UpperCamelCase_    , normalized=UpperCamelCase_ )
               if isinstance(UpperCamelCase_    , UpperCamelCase_ )
               else mask_token
           )

           super().__init__(
               UpperCamelCase_    , tokenizer_file=UpperCamelCase_    , do_lower_case=UpperCamelCase_    , remove_space=UpperCamelCase_    , keep_accents=UpperCamelCase_    , unk_token=UpperCamelCase_    , sep_token=UpperCamelCase_    , pad_token=UpperCamelCase_    , cls_token=UpperCamelCase_    , mask_token=UpperCamelCase_    , **UpperCamelCase_    , )

           lowerCAmelCase    :		Optional[int]				=      do_lower_case
           lowerCAmelCase    :		str				=      remove_space
           lowerCAmelCase    :		Any				=      keep_accents
           lowerCAmelCase    :		int				=      vocab_file
           lowerCAmelCase    :		List[str]				=      False if not self.vocab_file else True






     def    lowerCamelCase__   (		self     :    List[Any]    , UpperCamelCase_     :    List[int]    , UpperCamelCase_     :    Optional[List[int]] = None ):
           lowerCAmelCase    :		Optional[int]				=      [self.sep_token_id]
           lowerCAmelCase    :		Optional[Any]				=      [self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep






     def    lowerCamelCase__   (		self     :    List[str]    , UpperCamelCase_     :    List[int]    , UpperCamelCase_     :    Optional[List[int]] = None ):
           lowerCAmelCase    :		List[str]				=      [self.sep_token_id]
           lowerCAmelCase    :		Optional[Any]				=      [self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






     def    lowerCamelCase__   (		self     :    List[Any]    , UpperCamelCase_     :    str    , UpperCamelCase_     :    Optional[str] = None ):
           if not os.path.isdir(UpperCamelCase_ ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
                 return
           lowerCAmelCase    :		str				=      os.path.join(
               UpperCamelCase_    , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

           if os.path.abspath(self.vocab_file ) != os.path.abspath(UpperCamelCase_ ):
                 copyfile(self.vocab_file    , UpperCamelCase_ )

           return (out_vocab_file,)


