






"""simple docstring"""






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase_   : Any   			= logging.getLogger(__name__)



def 					_A			() ->       Optional[Any]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_      :  Tuple    						=		argparse.ArgumentParser(
        description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''							)
    parser.add_argument(
        '''--dataset_name'''    ,				type=__lowerCamelCase    ,				default='''wikitext'''    ,				help='''Name of the training. Explore datasets at: hf.co/datasets.'''    ,				)
    parser.add_argument(
        '''--dataset_config'''    ,				type=__lowerCamelCase    ,				default='''wikitext-103-raw-v1'''    ,				help='''Configuration name of the dataset.'''							)
    parser.add_argument(
        '''--tokenizer_name_or_path'''    ,				type=__lowerCamelCase    ,				default='''sayakpaul/unigram-tokenizer-wikitext'''    ,				help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''    ,				)
    parser.add_argument(
        '''--shard_size'''    ,				type=__lowerCamelCase    ,				default=10_00    ,				help='''Number of entries to go in a single shard.'''    ,				)
    parser.add_argument('''--split'''    ,				type=__lowerCamelCase    ,				default='''train'''    ,				choices=['''train''', '''test''', '''validation''']							)
    parser.add_argument(
        '''--limit'''    ,				default=__lowerCamelCase    ,				type=__lowerCamelCase    ,				help='''Limit the number of shards (used for debugging).'''    ,				)
    parser.add_argument(
        '''--max_length'''    ,				type=__lowerCamelCase    ,				default=5_12    ,				help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
        ''' sequence length that is a multiple of 8.'''    ,				)
    parser.add_argument(
        '''--output_dir'''    ,				default='''tf-tpu'''    ,				type=__lowerCamelCase    ,				help='''Output directory where the TFRecord shards will be saved. If the'''
        ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
        ''' shards will be directly saved to a Google Cloud Storage bucket.'''    ,				)

    SCREAMING_SNAKE_CASE_      :  Dict    						=		parser.parse_args()
    return args



def 					_A			(__a							) ->       Any:
    """simple docstring"""






    def fn(__a							):
        return tokenizer(examples['''text''']							)

    return fn



def 					_A			(__a							) ->       List[Any]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[]
    for i in range(len(tokenized_data['''input_ids''']							)							):
        SCREAMING_SNAKE_CASE_      :  str    						=		{
            "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]							)							),
            "attention_mask": tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]							)							),
        }
        SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		tf.train.Features(feature=__lowerCamelCase							)
        SCREAMING_SNAKE_CASE_      :  List[str]    						=		tf.train.Example(features=__lowerCamelCase							)
        SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		example.SerializeToString()
        records.append(__lowerCamelCase							)
    return records



def 					_A			(__a							) ->       Optional[Any]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_      :  Any    						=		datasets.load_dataset(args.dataset_name    ,				args.dataset_config    ,				split=args.split							)

    if args.limit is not None:
        SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		min(len(__lowerCamelCase							)    ,				args.limit							)
        SCREAMING_SNAKE_CASE_      :  Tuple    						=		dataset.select(range(__lowerCamelCase							)							)
        print(f'Limiting the dataset to {args.limit} entries.'							)

    SCREAMING_SNAKE_CASE_      :  Tuple    						=		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir							):
            os.makedirs(args.output_dir							)
        SCREAMING_SNAKE_CASE_      :  List[Any]    						=		os.path.join(args.output_dir    ,				args.split							)
        if not os.path.exists(__lowerCamelCase							):
            os.makedirs(__lowerCamelCase							)
    else:
        SCREAMING_SNAKE_CASE_      :  Tuple    						=		os.path.join(args.output_dir    ,				args.split							)

    # Tokenize the whole dataset at once.
    SCREAMING_SNAKE_CASE_      :  Any    						=		tokenize_function(__lowerCamelCase							)
    SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		dataset.map(__lowerCamelCase    ,				batched=__lowerCamelCase    ,				num_proc=4    ,				remove_columns=['''text''']							)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(__a							):
        # Concatenate all texts.
        SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		{k: sum(examples[k]    ,				[]							) for k in examples.keys()}
        SCREAMING_SNAKE_CASE_      :  str    						=		len(concatenated_examples[list(examples.keys()							)[0]]							)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        SCREAMING_SNAKE_CASE_      :  Any    						=		(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		{
            k: [t[i : i + args.max_length] for i in range(0    ,				__lowerCamelCase    ,				args.max_length							)]
            for k, t in concatenated_examples.items()
        }
        return result

    SCREAMING_SNAKE_CASE_      :  Any    						=		dataset_tokenized.map(__lowerCamelCase    ,				batched=__lowerCamelCase    ,				batch_size=10_00    ,				num_proc=4							)

    SCREAMING_SNAKE_CASE_      :  Dict    						=		0
    SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		0
    for shard in range(0    ,				len(__lowerCamelCase							)    ,				args.shard_size							):
        SCREAMING_SNAKE_CASE_      :  int    						=		grouped_dataset[shard : shard + args.shard_size]
        SCREAMING_SNAKE_CASE_      :  Any    						=		len(dataset_snapshot['''input_ids''']							)
        SCREAMING_SNAKE_CASE_      :  List[Any]    						=		os.path.join(__lowerCamelCase    ,				f'dataset-{shard_count}-{records_containing}.tfrecord'							)
        SCREAMING_SNAKE_CASE_      :  Tuple    						=		get_serialized_examples(__lowerCamelCase							)

        with tf.io.TFRecordWriter(__lowerCamelCase							) as out_file:
            for i in range(len(__lowerCamelCase							)							):
                SCREAMING_SNAKE_CASE_      :  Dict    						=		serialized_examples[i]
                out_file.write(__lowerCamelCase							)
            print('''Wrote file {} containing {} records'''.format(__lowerCamelCase    ,				__lowerCamelCase							)							)

        shard_count += 1
        total_records += records_containing

    with open(f'split-{args.split}-records-count.txt'    ,				'''w'''							) as f:
        print(f'Total {args.split} records: {total_records}'    ,				file=__lowerCamelCase							)


if __name__ == "__main__":
  UpperCAmelCase_   : Union[str, Any]   			= parse_args()
  main(args)




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


if is_sentencepiece_available():
   from ..ta.tokenization_ta import TaTokenizer
else:
   from ...utils.dummy_sentencepiece_objects import TaTokenizer

__lowerCamelCase							=       TaTokenizer

if is_tokenizers_available():
   from ..ta.tokenization_ta_fast import TaTokenizerFast
else:
   from ...utils.dummy_tokenizers_objects import TaTokenizerFast

__lowerCamelCase							=       TaTokenizerFast

__lowerCamelCase							=       {"""configuration_mt5""": ["""MT5Config""", """MT5OnnxConfig"""]}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       [
       """MT5EncoderModel""",
       """MT5ForConditionalGeneration""",
       """MT5ForQuestionAnswering""",
       """MT5Model""",
       """MT5PreTrainedModel""",
       """MT5Stack""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       ["""TFMT5EncoderModel""", """TFMT5ForConditionalGeneration""", """TFMT5Model"""]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase							=       ["""FlaxMT5EncoderModel""", """FlaxMT5ForConditionalGeneration""", """FlaxMT5Model"""]


if TYPE_CHECKING:
   from .configuration_mta import MTaConfig, MTaOnnxConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mta import (
          MTaEncoderModel,
          MTaForConditionalGeneration,
          MTaForQuestionAnswering,
          MTaModel,
          MTaPreTrainedModel,
          MTaStack,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_mta import TFMTaEncoderModel, TFMTaForConditionalGeneration, TFMTaModel

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_mta import FlaxMTaEncoderModel, FlaxMTaForConditionalGeneration, FlaxMTaModel

else:
   import sys

   __lowerCamelCase							=       _LazyModule(
       __name__,
       globals()["""__file__"""],
       _import_structure,
       extra_objects={"""MT5Tokenizer""": MTaTokenizer, """MT5TokenizerFast""": MTaTokenizerFast},
       module_spec=__spec__,
   )





