


'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase							:   int  						=	logging.getLogger(__name__)

def _SCREAMING_SNAKE_CASE      ()       ->       List[Any]:






       """simple docstring"""







       lowercase__      			= argparse.ArgumentParser(
           description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''       )
       parser.add_argument(
           '''--dataset_name'''  , type=A  , default='''wikitext'''  , help='''Name of the training. Explore datasets at: hf.co/datasets.'''  , )
       parser.add_argument(
           '''--dataset_config'''  , type=A  , default='''wikitext-103-raw-v1'''  , help='''Configuration name of the dataset.'''       )
       parser.add_argument(
           '''--tokenizer_name_or_path'''  , type=A  , default='''sayakpaul/unigram-tokenizer-wikitext'''  , help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''  , )
       parser.add_argument(
           '''--shard_size'''  , type=A  , default=1_000  , help='''Number of entries to go in a single shard.'''  , )
       parser.add_argument('''--split'''  , type=A  , default='''train'''  , choices=['''train''', '''test''', '''validation''']       )
       parser.add_argument(
           '''--limit'''  , default=A  , type=A  , help='''Limit the number of shards (used for debugging).'''  , )
       parser.add_argument(
           '''--max_length'''  , type=A  , default=512  , help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
           ''' sequence length that is a multiple of 8.'''  , )
       parser.add_argument(
           '''--output_dir'''  , default='''tf-tpu'''  , type=A  , help='''Output directory where the TFRecord shards will be saved. If the'''
           ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
           ''' shards will be directly saved to a Google Cloud Storage bucket.'''  , )

       lowercase__      			= parser.parse_args()
       return args

def _SCREAMING_SNAKE_CASE      (A       )       ->       str:






       """simple docstring"""







       def fn(A       ):
              return tokenizer(examples['''text''']       )

       return fn

def _SCREAMING_SNAKE_CASE      (A       )       ->       Union[str, Any]:






       """simple docstring"""







       lowercase__      			= []
       for i in range(len(tokenized_data['''input_ids''']       )       ):
              lowercase__      			= {
                  '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]       )       ),
                  '''attention_mask''': tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]       )       ),
              }
              lowercase__      			= tf.train.Features(feature=A       )
              lowercase__      			= tf.train.Example(features=A       )
              lowercase__      			= example.SerializeToString()
              records.append(A       )
       return records

def _SCREAMING_SNAKE_CASE      (A       )       ->       Tuple:






       """simple docstring"""







       lowercase__      			= datasets.load_dataset(args.dataset_name  , args.dataset_config  , split=args.split       )

       if args.limit is not None:
              lowercase__      			= min(len(A       )  , args.limit       )
              lowercase__      			= dataset.select(range(A       )       )
              print(f"Limiting the dataset to {args.limit} entries."       )

       lowercase__      			= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir       ):
                     os.makedirs(args.output_dir       )
              lowercase__      			= os.path.join(args.output_dir  , args.split       )
              if not os.path.exists(A       ):
                     os.makedirs(A       )
       else:
              lowercase__      			= os.path.join(args.output_dir  , args.split       )

       # Tokenize the whole dataset at once.
       lowercase__      			= tokenize_function(A       )
       lowercase__      			= dataset.map(A  , batched=A  , num_proc=4  , remove_columns=['''text''']       )

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(A       ):
              # Concatenate all texts.
              lowercase__      			= {k: sum(examples[k]  , []       ) for k in examples.keys()}
              lowercase__      			= len(concatenated_examples[list(examples.keys()       )[0]]       )
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              lowercase__      			= (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              lowercase__      			= {
                  k: [t[i : i + args.max_length] for i in range(0  , A  , args.max_length       )]
                  for k, t in concatenated_examples.items()
              }
              return result

       lowercase__      			= dataset_tokenized.map(A  , batched=A  , batch_size=1_000  , num_proc=4       )

       lowercase__      			= 0
       lowercase__      			= 0
       for shard in range(0  , len(A       )  , args.shard_size       ):
              lowercase__      			= grouped_dataset[shard : shard + args.shard_size]
              lowercase__      			= len(dataset_snapshot['''input_ids''']       )
              lowercase__      			= os.path.join(A  , f"dataset-{shard_count}-{records_containing}.tfrecord"       )
              lowercase__      			= get_serialized_examples(A       )

              with tf.io.TFRecordWriter(A       ) as out_file:
                     for i in range(len(A       )       ):
                            lowercase__      			= serialized_examples[i]
                            out_file.write(A       )
                     print('''Wrote file {} containing {} records'''.format(A  , A       )       )

              shard_count += 1
              total_records += records_containing

       with open(f"split-{args.split}-records-count.txt"  , '''w'''       ) as f:
              print(f"Total {args.split} records: {total_records}"  , file=A       )


if __name__ == "__main__":
 lowerCamelCase							:   List[Any]  						=	parse_args()
 main(args)








'''simple docstring'''

# Lint as: python3
import itertools
import os
import re


lowerCamelCase							:   Any  						=	re.compile(R'([A-Z]+)([A-Z][a-z])')
lowerCamelCase							:   str  						=	re.compile(R'([a-z\d])([A-Z])')

lowerCamelCase							:   Optional[int]  						=	re.compile(R'(?<!_)_(?!_)')
lowerCamelCase							:   List[Any]  						=	re.compile(R'(_{2,})')

lowerCamelCase							:   str  						=	R'^\w+(\.\w+)*$'

lowerCamelCase							:   Dict  						=	R'<>:/\|?*'

def _SCREAMING_SNAKE_CASE      (A       )       ->       Any:






       """simple docstring"""







       lowercase__      			= _uppercase_uppercase_re.sub(R'''\1_\2'''  , A       )
       lowercase__      			= _lowercase_uppercase_re.sub(R'''\1_\2'''  , A       )
       return name.lower()

def _SCREAMING_SNAKE_CASE      (A       )       ->       Tuple:






       """simple docstring"""







       lowercase__      			= _single_underscore_re.split(A       )
       lowercase__      			= [_multiple_underscores_re.split(A       ) for n in name]
       return "".join(n.capitalize() for n in itertools.chain.from_iterable(A       ) if n != ''''''       )

def _SCREAMING_SNAKE_CASE      (A       )       ->       Tuple:






       """simple docstring"""







       if os.path.basename(A       ) != name:
              raise ValueError(f"Should be a dataset name, not a path: {name}"       )
       return camelcase_to_snakecase(A       )

def _SCREAMING_SNAKE_CASE      (A  , A       )       ->       Optional[Any]:






       """simple docstring"""







       if os.path.basename(A       ) != name:
              raise ValueError(f"Should be a dataset name, not a path: {name}"       )
       if not re.match(_split_re  , A       ):
              raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'."       )
       return f"{filename_prefix_for_name(A       )}-{split}"

def _SCREAMING_SNAKE_CASE      (A  , A  , A  , A=None       )       ->       List[str]:






       """simple docstring"""







       lowercase__      			= filename_prefix_for_split(A  , A       )
       if filetype_suffix:
              prefix += f".{filetype_suffix}"
       lowercase__      			= os.path.join(A  , A       )
       return f"{filepath}*"

def _SCREAMING_SNAKE_CASE      (A  , A  , A  , A=None  , A=None       )       ->       Optional[Any]:






       """simple docstring"""







       lowercase__      			= filename_prefix_for_split(A  , A       )
       lowercase__      			= os.path.join(A  , A       )

       if shard_lengths:
              lowercase__      			= len(A       )
              lowercase__      			= [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(A       )]
              if filetype_suffix:
                     lowercase__      			= [filename + f".{filetype_suffix}" for filename in filenames]
              return filenames
       else:
              lowercase__      			= prefix
              if filetype_suffix:
                     filename += f".{filetype_suffix}"
              return [filename]




