


"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case__						:   Optional[Any]			 = logging.getLogger(__name__)


def    _snake_case  (  ):
      lowerCAmelCase    :		Tuple				=      argparse.ArgumentParser(
          description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''	)
      parser.add_argument(
          '''--dataset_name'''       ,      type=_snake_case       ,      default='''wikitext'''       ,      help='''Name of the training. Explore datasets at: hf.co/datasets.'''       ,      )
      parser.add_argument(
          '''--dataset_config'''       ,      type=_snake_case       ,      default='''wikitext-103-raw-v1'''       ,      help='''Configuration name of the dataset.'''	)
      parser.add_argument(
          '''--tokenizer_name_or_path'''       ,      type=_snake_case       ,      default='''sayakpaul/unigram-tokenizer-wikitext'''       ,      help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''       ,      )
      parser.add_argument(
          '''--shard_size'''       ,      type=_snake_case       ,      default=1000       ,      help='''Number of entries to go in a single shard.'''       ,      )
      parser.add_argument('''--split'''       ,      type=_snake_case       ,      default='''train'''       ,      choices=['''train''', '''test''', '''validation''']	)
      parser.add_argument(
          '''--limit'''       ,      default=_snake_case       ,      type=_snake_case       ,      help='''Limit the number of shards (used for debugging).'''       ,      )
      parser.add_argument(
          '''--max_length'''       ,      type=_snake_case       ,      default=512       ,      help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
          ''' sequence length that is a multiple of 8.'''       ,      )
      parser.add_argument(
          '''--output_dir'''       ,      default='''tf-tpu'''       ,      type=_snake_case       ,      help='''Output directory where the TFRecord shards will be saved. If the'''
          ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
          ''' shards will be directly saved to a Google Cloud Storage bucket.'''       ,      )

      lowerCAmelCase    :		Optional[Any]				=      parser.parse_args()
      return args


def    _snake_case  (  _snake_case :	Optional[Any]	):
      def fn(_snake_case :	Optional[int]	):
            return tokenizer(examples['''text''']	)

      return fn


def    _snake_case  (  _snake_case :	List[Any]	):
      lowerCAmelCase    :		Optional[Any]				=      []
      for i in range(len(tokenized_data['''input_ids''']	)	):
            lowerCAmelCase    :		Optional[int]				=      {
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]	)	),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]	)	),
            }
            lowerCAmelCase    :		Optional[Any]				=      tf.train.Features(feature=_snake_case	)
            lowerCAmelCase    :		int				=      tf.train.Example(features=_snake_case	)
            lowerCAmelCase    :		Dict				=      example.SerializeToString()
            records.append(_snake_case	)
      return records


def    _snake_case  (  _snake_case :	List[Any]	):
      lowerCAmelCase    :		str				=      datasets.load_dataset(args.dataset_name       ,      args.dataset_config       ,      split=args.split	)

      if args.limit is not None:
            lowerCAmelCase    :		Dict				=      min(len(_snake_case	)       ,      args.limit	)
            lowerCAmelCase    :		List[Any]				=      dataset.select(range(_snake_case	)	)
            print(f'''Limiting the dataset to {args.limit} entries.'''	)

      lowerCAmelCase    :		Any				=      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path	)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir	):
                  os.makedirs(args.output_dir	)
            lowerCAmelCase    :		Union[str, Any]				=      os.path.join(args.output_dir       ,      args.split	)
            if not os.path.exists(_snake_case	):
                  os.makedirs(_snake_case	)
      else:
            lowerCAmelCase    :		int				=      os.path.join(args.output_dir       ,      args.split	)

      # Tokenize the whole dataset at once.
      lowerCAmelCase    :		Optional[int]				=      tokenize_function(_snake_case	)
      lowerCAmelCase    :		Tuple				=      dataset.map(_snake_case       ,      batched=_snake_case       ,      num_proc=4       ,      remove_columns=['''text''']	)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(_snake_case :	Tuple	):
            # Concatenate all texts.
            lowerCAmelCase    :		int				=      {k: sum(examples[k]       ,      []	) for k in examples.keys()}
            lowerCAmelCase    :		int				=      len(concatenated_examples[list(examples.keys()	)[0]]	)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            lowerCAmelCase    :		Optional[Any]				=      (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            lowerCAmelCase    :		Any				=      {
                k: [t[i : i + args.max_length] for i in range(0       ,      _snake_case       ,      args.max_length	)]
                for k, t in concatenated_examples.items()
            }
            return result

      lowerCAmelCase    :		Tuple				=      dataset_tokenized.map(_snake_case       ,      batched=_snake_case       ,      batch_size=1000       ,      num_proc=4	)

      lowerCAmelCase    :		str				=      0
      lowerCAmelCase    :		Any				=      0
      for shard in range(0       ,      len(_snake_case	)       ,      args.shard_size	):
            lowerCAmelCase    :		Dict				=      grouped_dataset[shard : shard + args.shard_size]
            lowerCAmelCase    :		List[str]				=      len(dataset_snapshot['''input_ids''']	)
            lowerCAmelCase    :		List[str]				=      os.path.join(_snake_case       ,      f'''dataset-{shard_count}-{records_containing}.tfrecord'''	)
            lowerCAmelCase    :		Union[str, Any]				=      get_serialized_examples(_snake_case	)

            with tf.io.TFRecordWriter(_snake_case	) as out_file:
                  for i in range(len(_snake_case	)	):
                        lowerCAmelCase    :		Dict				=      serialized_examples[i]
                        out_file.write(_snake_case	)
                  print('''Wrote file {} containing {} records'''.format(_snake_case       ,      _snake_case	)	)

            shard_count += 1
            total_records += records_containing

      with open(f'''split-{args.split}-records-count.txt'''       ,      '''w'''	) as f:
            print(f'''Total {args.split} records: {total_records}'''       ,      file=_snake_case	)


if __name__ == "__main__":
 snake_case__						:   str			 = parse_args()
 main(args)






"""simple docstring"""





from __future__ import annotations

from decimal import Decimal

from numpy import array


def    _snake_case  (  _snake_case :	list[list[float]]	):
      lowerCAmelCase    :		str				=      Decimal

      # Check if the provided matrix has 2 rows and 2 columns
      # since this implementation only works for 2x2 matrices
      if len(_snake_case	) == 2 and len(matrix[0]	) == 2 and len(matrix[1]	) == 2:
            # Calculate the determinant of the matrix
            lowerCAmelCase    :		int				=      float(
                d(matrix[0][0]	) * d(matrix[1][1]	) - d(matrix[1][0]	) * d(matrix[0][1]	)	)
            if determinant == 0:
                  raise ValueError('''This matrix has no inverse.'''	)

            # Creates a copy of the matrix with swapped positions of the elements
            lowerCAmelCase    :		Optional[int]				=      [[0.0, 0.0], [0.0, 0.0]]
            lowerCAmelCase,					lowerCAmelCase    :		List[Any]				=      matrix[1][1], matrix[0][0]
            lowerCAmelCase,					lowerCAmelCase    :		Union[str, Any]				=      -matrix[1][0], -matrix[0][1]

            # Calculate the inverse of the matrix
            return [
                [(float(d(_snake_case	)	) / determinant) or 0.0 for n in row] for row in swapped_matrix
            ]
      elif (
          len(_snake_case	) == 3
          and len(matrix[0]	) == 3
          and len(matrix[1]	) == 3
          and len(matrix[2]	) == 3
      ):
            # Calculate the determinant of the matrix using Sarrus rule
            lowerCAmelCase    :		int				=      float(
                (
                    (d(matrix[0][0]	) * d(matrix[1][1]	) * d(matrix[2][2]	))
                    + (d(matrix[0][1]	) * d(matrix[1][2]	) * d(matrix[2][0]	))
                    + (d(matrix[0][2]	) * d(matrix[1][0]	) * d(matrix[2][1]	))
                )
                - (
                    (d(matrix[0][2]	) * d(matrix[1][1]	) * d(matrix[2][0]	))
                    + (d(matrix[0][1]	) * d(matrix[1][0]	) * d(matrix[2][2]	))
                    + (d(matrix[0][0]	) * d(matrix[1][2]	) * d(matrix[2][1]	))
                )	)
            if determinant == 0:
                  raise ValueError('''This matrix has no inverse.'''	)

            # Creating cofactor matrix
            lowerCAmelCase    :		Dict				=      [
                [d(0.0	), d(0.0	), d(0.0	)],
                [d(0.0	), d(0.0	), d(0.0	)],
                [d(0.0	), d(0.0	), d(0.0	)],
            ]
            lowerCAmelCase    :		List[str]				=      (d(matrix[1][1]	) * d(matrix[2][2]	)) - (
                d(matrix[1][2]	) * d(matrix[2][1]	)
            )
            lowerCAmelCase    :		Dict				=      -(
                (d(matrix[1][0]	) * d(matrix[2][2]	)) - (d(matrix[1][2]	) * d(matrix[2][0]	))
            )
            lowerCAmelCase    :		str				=      (d(matrix[1][0]	) * d(matrix[2][1]	)) - (
                d(matrix[1][1]	) * d(matrix[2][0]	)
            )
            lowerCAmelCase    :		Any				=      -(
                (d(matrix[0][1]	) * d(matrix[2][2]	)) - (d(matrix[0][2]	) * d(matrix[2][1]	))
            )
            lowerCAmelCase    :		Any				=      (d(matrix[0][0]	) * d(matrix[2][2]	)) - (
                d(matrix[0][2]	) * d(matrix[2][0]	)
            )
            lowerCAmelCase    :		Optional[int]				=      -(
                (d(matrix[0][0]	) * d(matrix[2][1]	)) - (d(matrix[0][1]	) * d(matrix[2][0]	))
            )
            lowerCAmelCase    :		Optional[int]				=      (d(matrix[0][1]	) * d(matrix[1][2]	)) - (
                d(matrix[0][2]	) * d(matrix[1][1]	)
            )
            lowerCAmelCase    :		Dict				=      -(
                (d(matrix[0][0]	) * d(matrix[1][2]	)) - (d(matrix[0][2]	) * d(matrix[1][0]	))
            )
            lowerCAmelCase    :		List[Any]				=      (d(matrix[0][0]	) * d(matrix[1][1]	)) - (
                d(matrix[0][1]	) * d(matrix[1][0]	)
            )

            # Transpose the cofactor matrix (Adjoint matrix)
            lowerCAmelCase    :		str				=      array(_snake_case	)
            for i in range(3	):
                  for j in range(3	):
                        lowerCAmelCase    :		Optional[Any]				=      cofactor_matrix[j][i]

        # Inverse of the matrix using the formula (1/determinant) * adjoint matrix
            lowerCAmelCase    :		Tuple				=      array(_snake_case	)
            for i in range(3	):
                  for j in range(3	):
                        inverse_matrix[i][j] /= d(_snake_case	)

        # Calculate the inverse of the matrix
            return [[float(d(_snake_case	)	) or 0.0 for n in row] for row in inverse_matrix]
      raise ValueError('''Please provide a matrix of size 2x2 or 3x3.'''	)


