





"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE__					=   logging.getLogger(__name__)


def 			UpperCAmelCase__							(  ):







      '''simple docstring'''




      lowerCAmelCase  						=       argparse.ArgumentParser(
          description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""			)
      parser.add_argument(
          """--dataset_name"""							,						type=SCREAMING_SNAKE_CASE							,						default="""wikitext"""							,						help="""Name of the training. Explore datasets at: hf.co/datasets."""							,						)
      parser.add_argument(
          """--dataset_config"""							,						type=SCREAMING_SNAKE_CASE							,						default="""wikitext-103-raw-v1"""							,						help="""Configuration name of the dataset."""			)
      parser.add_argument(
          """--tokenizer_name_or_path"""							,						type=SCREAMING_SNAKE_CASE							,						default="""sayakpaul/unigram-tokenizer-wikitext"""							,						help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""							,						)
      parser.add_argument(
          """--shard_size"""							,						type=SCREAMING_SNAKE_CASE							,						default=10_00							,						help="""Number of entries to go in a single shard."""							,						)
      parser.add_argument("""--split"""							,						type=SCREAMING_SNAKE_CASE							,						default="""train"""							,						choices=["""train""", """test""", """validation"""]			)
      parser.add_argument(
          """--limit"""							,						default=SCREAMING_SNAKE_CASE							,						type=SCREAMING_SNAKE_CASE							,						help="""Limit the number of shards (used for debugging)."""							,						)
      parser.add_argument(
          """--max_length"""							,						type=SCREAMING_SNAKE_CASE							,						default=5_12							,						help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
          """ sequence length that is a multiple of 8."""							,						)
      parser.add_argument(
          """--output_dir"""							,						default="""tf-tpu"""							,						type=SCREAMING_SNAKE_CASE							,						help="""Output directory where the TFRecord shards will be saved. If the"""
          """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
          """ shards will be directly saved to a Google Cloud Storage bucket."""							,						)

      lowerCAmelCase  						=       parser.parse_args()
      return args


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						List[Any]			):







      '''simple docstring'''




      def fn(SCREAMING_SNAKE_CASE     :						Optional[int]			):
            return tokenizer(examples["""text"""]			)

      return fn


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						Union[str, Any]			):







      '''simple docstring'''




      lowerCAmelCase  						=       []
      for i in range(len(tokenized_data["""input_ids"""]			)			):
            lowerCAmelCase  						=       {
                """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]			)			),
                """attention_mask""": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]			)			),
            }
            lowerCAmelCase  						=       tf.train.Features(feature=SCREAMING_SNAKE_CASE			)
            lowerCAmelCase  						=       tf.train.Example(features=SCREAMING_SNAKE_CASE			)
            lowerCAmelCase  						=       example.SerializeToString()
            records.append(SCREAMING_SNAKE_CASE			)
      return records


def 			UpperCAmelCase__							(  SCREAMING_SNAKE_CASE     :						str			):







      '''simple docstring'''




      lowerCAmelCase  						=       datasets.load_dataset(args.dataset_name							,						args.dataset_config							,						split=args.split			)

      if args.limit is not None:
            lowerCAmelCase  						=       min(len(SCREAMING_SNAKE_CASE			)							,						args.limit			)
            lowerCAmelCase  						=       dataset.select(range(SCREAMING_SNAKE_CASE			)			)
            print(F'Limiting the dataset to {args.limit} entries.'			)

      lowerCAmelCase  						=       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir			):
                  os.makedirs(args.output_dir			)
            lowerCAmelCase  						=       os.path.join(args.output_dir							,						args.split			)
            if not os.path.exists(SCREAMING_SNAKE_CASE			):
                  os.makedirs(SCREAMING_SNAKE_CASE			)
      else:
            lowerCAmelCase  						=       os.path.join(args.output_dir							,						args.split			)

      # Tokenize the whole dataset at once.
      lowerCAmelCase  						=       tokenize_function(SCREAMING_SNAKE_CASE			)
      lowerCAmelCase  						=       dataset.map(SCREAMING_SNAKE_CASE							,						batched=SCREAMING_SNAKE_CASE							,						num_proc=4							,						remove_columns=["""text"""]			)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(SCREAMING_SNAKE_CASE     :						Any			):
            # Concatenate all texts.
            lowerCAmelCase  						=       {k: sum(examples[k]							,						[]			) for k in examples.keys()}
            lowerCAmelCase  						=       len(concatenated_examples[list(examples.keys()			)[0]]			)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            lowerCAmelCase  						=       (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            lowerCAmelCase  						=       {
                k: [t[i : i + args.max_length] for i in range(0							,						SCREAMING_SNAKE_CASE							,						args.max_length			)]
                for k, t in concatenated_examples.items()
            }
            return result

      lowerCAmelCase  						=       dataset_tokenized.map(SCREAMING_SNAKE_CASE							,						batched=SCREAMING_SNAKE_CASE							,						batch_size=10_00							,						num_proc=4			)

      lowerCAmelCase  						=       0
      lowerCAmelCase  						=       0
      for shard in range(0							,						len(SCREAMING_SNAKE_CASE			)							,						args.shard_size			):
            lowerCAmelCase  						=       grouped_dataset[shard : shard + args.shard_size]
            lowerCAmelCase  						=       len(dataset_snapshot["""input_ids"""]			)
            lowerCAmelCase  						=       os.path.join(SCREAMING_SNAKE_CASE							,						F'dataset-{shard_count}-{records_containing}.tfrecord'			)
            lowerCAmelCase  						=       get_serialized_examples(SCREAMING_SNAKE_CASE			)

            with tf.io.TFRecordWriter(SCREAMING_SNAKE_CASE			) as out_file:
                  for i in range(len(SCREAMING_SNAKE_CASE			)			):
                        lowerCAmelCase  						=       serialized_examples[i]
                        out_file.write(SCREAMING_SNAKE_CASE			)
                  print("""Wrote file {} containing {} records""".format(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE			)			)

            shard_count += 1
            total_records += records_containing

      with open(F'split-{args.split}-records-count.txt'							,						"""w"""			) as f:
            print(F'Total {args.split} records: {total_records}'							,						file=SCREAMING_SNAKE_CASE			)


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__					=   parse_args()
       main(args)














'''simple docstring'''







_A	:			Union[str, Any]         =range(2, 20 + 1)
_A	:			List[str]         =[10**k for k in range(ks[-1] + 1)]
_A	:			dict[int, dict[int, list[list[int]]]]         ={}





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       List[Any]:
							lowerCamelCase__					:	List[str]         =       sum(a_i[j] for j in range(UpperCamelCase				,			len(UpperCamelCase      )      )      )
							lowerCamelCase__					:	int         =       sum(a_i[j] * base[j] for j in range(min(len(UpperCamelCase      )				,			UpperCamelCase      )      )      )

							lowerCamelCase__  ,		lowerCamelCase__					:	Dict         =       0, 0
							lowerCamelCase__					:	List[str]         =       n - i

							lowerCamelCase__					:	Optional[Any]         =       memo.get(UpperCamelCase      )

							if sub_memo is not None:
														lowerCamelCase__					:	str         =       sub_memo.get(UpperCamelCase      )

														if jumps is not None and len(UpperCamelCase      ) > 0:
																					# find and make the largest jump without going over
																					lowerCamelCase__					:	Optional[Any]         =       -1
																					for _k in range(len(UpperCamelCase      ) - 1				,			-1				,			-1      ):
																												if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																																			lowerCamelCase__					:	Dict         =       _k
																																			break

																					if max_jump >= 0:
																												lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	str         =       jumps[max_jump]
																												# since the difference between jumps is cached, add c
																												lowerCamelCase__					:	Dict         =       diff + c
																												for j in range(min(UpperCamelCase				,			len(UpperCamelCase      )      )      ):
																																			lowerCamelCase__  ,		lowerCamelCase__					:	List[Any]         =       divmod(UpperCamelCase				,			10      )
																												if new_c > 0:
																																			add(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )

														else:
																					lowerCamelCase__					:	Any         =       []
							else:
														lowerCamelCase__					:	str         =       {c: []}
														lowerCamelCase__					:	Tuple         =       sub_memo

							if dn >= max_dn or c + diff >= base[k]:
														return diff, dn

							if k > ks[0]:
														while True:
																					# keep doing smaller jumps
																					lowerCamelCase__  ,		lowerCamelCase__					:	Dict         =       next_term(UpperCamelCase				,			k - 1				,			i + dn				,			UpperCamelCase      )
																					diff += _diff
																					dn += terms_jumped

																					if dn >= max_dn or c + diff >= base[k]:
																												break
							else:
														# would be too small a jump, just compute sequential terms instead
														lowerCamelCase__  ,		lowerCamelCase__					:	Optional[int]         =       compute(UpperCamelCase				,			UpperCamelCase				,			i + dn				,			UpperCamelCase      )
														diff += _diff
														dn += terms_jumped

							lowerCamelCase__					:	str         =       sub_memo[c]

							# keep jumps sorted by # of terms skipped
							lowerCamelCase__					:	List[Any]         =       0
							while j < len(UpperCamelCase      ):
														if jumps[j][1] > dn:
																					break
														j += 1

							# cache the jump for this value digitsum(b) and c
							sub_memo[c].insert(UpperCamelCase				,			(diff, dn, k)      )
							return (diff, dn)





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       List[Any]:
							if i >= n:
														return 0, i
							if k > len(UpperCamelCase      ):
														a_i.extend([0 for _ in range(k - len(UpperCamelCase      )      )]      )

							# note: a_i -> b * 10^k + c
							# ds_b -> digitsum(b)
							# ds_c -> digitsum(c)
							lowerCamelCase__					:	Optional[Any]         =       i
							lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	Optional[int]         =       0, 0, 0
							for j in range(len(UpperCamelCase      )      ):
														if j >= k:
																					ds_b += a_i[j]
														else:
																					ds_c += a_i[j]

							while i < n:
														i += 1
														lowerCamelCase__					:	Optional[int]         =       ds_c + ds_b
														diff += addend
														lowerCamelCase__					:	int         =       0
														for j in range(UpperCamelCase      ):
																					lowerCamelCase__					:	str         =       a_i[j] + addend
																					lowerCamelCase__  ,		lowerCamelCase__					:	int         =       divmod(UpperCamelCase				,			10      )

																					ds_c += a_i[j]

														if addend > 0:
																					break

							if addend > 0:
														add(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )
							return diff, i - start_i





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       str:
							for j in range(UpperCamelCase				,			len(UpperCamelCase      )      ):
														lowerCamelCase__					:	List[Any]         =       digits[j] + addend
														if s >= 10:
																					lowerCamelCase__  ,		lowerCamelCase__					:	Union[str, Any]         =       divmod(UpperCamelCase				,			10      )
																					lowerCamelCase__					:	Any         =       addend // 10 + quotient
														else:
																					lowerCamelCase__					:	Any         =       s
																					lowerCamelCase__					:	Optional[Any]         =       addend // 10

														if addend == 0:
																					break

							while addend > 0:
														lowerCamelCase__  ,		lowerCamelCase__					:	Any         =       divmod(UpperCamelCase				,			10      )
														digits.append(UpperCamelCase      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase = 10**15      )      ->       int:
							lowerCamelCase__					:	Any         =       [1]
							lowerCamelCase__					:	List[str]         =       1
							lowerCamelCase__					:	Tuple         =       0
							while True:
														lowerCamelCase__  ,		lowerCamelCase__					:	Any         =       next_term(UpperCamelCase				,			20				,			i + dn				,			UpperCamelCase      )
														dn += terms_jumped
														if dn == n - i:
																					break

							lowerCamelCase__					:	Union[str, Any]         =       0
							for j in range(len(UpperCamelCase      )      ):
														a_n += digits[j] * 10**j
							return a_n


if __name__ == "__main__":
			print(F'{solution() = }')

