





'''simple docstring'''







import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase_					:    Dict      	=			logging.getLogger(__name__)






def  _lowerCamelCase				( )	->	Optional[int]:
				_a								= argparse.ArgumentParser(
				    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."						)
				parser.add_argument(
				    "--dataset_name"    ,							type=lowercase    ,							default="wikitext"    ,							help="Name of the training. Explore datasets at: hf.co/datasets."    ,							)
				parser.add_argument(
				    "--dataset_config"    ,							type=lowercase    ,							default="wikitext-103-raw-v1"    ,							help="Configuration name of the dataset."						)
				parser.add_argument(
				    "--tokenizer_name_or_path"    ,							type=lowercase    ,							default="sayakpaul/unigram-tokenizer-wikitext"    ,							help="Tokenizer identifier. Can be a local filepath or a Hub identifier."    ,							)
				parser.add_argument(
				    "--shard_size"    ,							type=lowercase    ,							default=1000    ,							help="Number of entries to go in a single shard."    ,							)
				parser.add_argument("--split"    ,							type=lowercase    ,							default="train"    ,							choices=["train", "test", "validation"]						)
				parser.add_argument(
				    "--limit"    ,							default=lowercase    ,							type=lowercase    ,							help="Limit the number of shards (used for debugging)."    ,							)
				parser.add_argument(
				    "--max_length"    ,							type=lowercase    ,							default=512    ,							help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
				    " sequence length that is a multiple of 8."    ,							)
				parser.add_argument(
				    "--output_dir"    ,							default="tf-tpu"    ,							type=lowercase    ,							help="Output directory where the TFRecord shards will be saved. If the"
				    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
				    " shards will be directly saved to a Google Cloud Storage bucket."    ,							)

				_a								= parser.parse_args()
				return args






def  _lowerCamelCase				( lowercase						:      List[Any]						)	->	Tuple:
				def fn(lowercase						:      Optional[Any]						):
								return tokenizer(examples["text"]						)

				return fn






def  _lowerCamelCase				( lowercase						:      List[Any]						)	->	Dict:
				_a								= []
				for i in range(len(tokenized_data["input_ids"]						)						):
								_a								= {
								    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]						)						),
								    "attention_mask": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]						)						),
								}
								_a								= tf.train.Features(feature=lowercase						)
								_a								= tf.train.Example(features=lowercase						)
								_a								= example.SerializeToString()
								records.append(lowercase						)
				return records






def  _lowerCamelCase				( lowercase						:      Dict						)	->	str:
				_a								= datasets.load_dataset(args.dataset_name    ,							args.dataset_config    ,							split=args.split						)

				if args.limit is not None:
								_a								= min(len(lowercase						)    ,							args.limit						)
								_a								= dataset.select(range(lowercase						)						)
								print(F'Limiting the dataset to {args.limit} entries.'						)

				_a								= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir						):
												os.makedirs(args.output_dir						)
								_a								= os.path.join(args.output_dir    ,							args.split						)
								if not os.path.exists(lowercase						):
												os.makedirs(lowercase						)
				else:
								_a								= os.path.join(args.output_dir    ,							args.split						)

				# Tokenize the whole dataset at once.
				_a								= tokenize_function(lowercase						)
				_a								= dataset.map(lowercase    ,							batched=lowercase    ,							num_proc=4    ,							remove_columns=["text"]						)

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowercase						:      Tuple						):
								# Concatenate all texts.
								_a								= {k: sum(examples[k]    ,							[]						) for k in examples.keys()}
								_a								= len(concatenated_examples[list(examples.keys()						)[0]]						)
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								_a								= (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								_a								= {
								    k: [t[i : i + args.max_length] for i in range(0    ,							lowercase    ,							args.max_length						)]
								    for k, t in concatenated_examples.items()
								}
								return result

				_a								= dataset_tokenized.map(lowercase    ,							batched=lowercase    ,							batch_size=1000    ,							num_proc=4						)

				_a								= 0
				_a								= 0
				for shard in range(0    ,							len(lowercase						)    ,							args.shard_size						):
								_a								= grouped_dataset[shard : shard + args.shard_size]
								_a								= len(dataset_snapshot["input_ids"]						)
								_a								= os.path.join(lowercase    ,							F'dataset-{shard_count}-{records_containing}.tfrecord'						)
								_a								= get_serialized_examples(lowercase						)

								with tf.io.TFRecordWriter(lowercase						) as out_file:
												for i in range(len(lowercase						)						):
																_a								= serialized_examples[i]
																out_file.write(lowercase						)
												print("Wrote file {} containing {} records".format(lowercase    ,							lowercase						)						)

								shard_count += 1
								total_records += records_containing

				with open(F'split-{args.split}-records-count.txt'    ,							"w"						) as f:
								print(F'Total {args.split} records: {total_records}'    ,							file=lowercase						)


if __name__ == "__main__":
					lowerCAmelCase_					:    Optional[Any]      	=			parse_args()
					main(args)







'''simple docstring'''







import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json






def  _lowerCamelCase				( lowercase						:      Dict						)	->	Any:
				_a								= filter(lambda lowercase						: p.requires_grad    ,							model.parameters()						)
				_a								= sum([np.prod(p.size()						) for p in model_parameters]						)
				return params


lowerCAmelCase_					:    int      	=			logging.getLogger(__name__)






def  _lowerCamelCase				( lowercase						:      List[Any]    ,							lowercase						:      Any						)	->	Any:
				if metric == "rouge2":
								_a								= "{val_avg_rouge2:.4f}-{step_count}"
				elif metric == "bleu":
								_a								= "{val_avg_bleu:.4f}-{step_count}"
				elif metric == "em":
								_a								= "{val_avg_em:.4f}-{step_count}"
				else:
								raise NotImplementedError(
								    F'seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this'
								    " function."						)

				_a								= ModelCheckpoint(
				    dirpath=lowercase    ,							filename=lowercase    ,							monitor=F'val_{metric}'    ,							mode="max"    ,							save_top_k=3    ,							every_n_epochs=1    ,							)
				return checkpoint_callback






def  _lowerCamelCase				( lowercase						:      Optional[int]    ,							lowercase						:      Optional[int]						)	->	Union[str, Any]:
				return EarlyStopping(
				    monitor=F'val_{metric}'    ,							mode="min" if "loss" in metric else "max"    ,							patience=lowercase    ,							verbose=lowercase    ,							)



class 				__SCREAMING_SNAKE_CASE  (pl.Callback      ):






				"""simple docstring"""







				def   UpperCamelCase__   (  self     :					Optional[int]					,				__a     :					str					,				__a     :					List[Any]					):
								_a								= {f'lr_group_{i}': param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups					)}
								pl_module.logger.log_metrics(__a					)




				@rank_zero_only
				def   UpperCamelCase__   (  self     :					Optional[int]					,				__a     :					pl.Trainer					,				__a     :					pl.LightningModule					,				__a     :					str					,				__a     :					Tuple=True					):
								logger.info(f'***** {type_path} results at step {trainer.global_step:05d} *****'					)
								_a								= trainer.callback_metrics
								trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}					)
								# Log results
								_a								= Path(pl_module.hparams.output_dir					)
								if type_path == "test":
												_a								= od / "test_results.txt"
												_a								= od / "test_generations.txt"
								else:
												# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
												# If people want this it will be easy enough to add back.
												_a								= od / f'{type_path}_results/{trainer.global_step:05d}.txt'
												_a								= od / f'{type_path}_generations/{trainer.global_step:05d}.txt'
												results_file.parent.mkdir(exist_ok=__a					)
												generations_file.parent.mkdir(exist_ok=__a					)
								with open(__a					,				"a+"					) as writer:
												for key in sorted(__a					):
																if key in ["log", "progress_bar", "preds"]:
																				continue
																_a								= metrics[key]
																if isinstance(__a					,				torch.Tensor					):
																				_a								= val.item()
																_a								= f'{key}: {val:.6f}\n'
																writer.write(__a					)

								if not save_generations:
												return

								if "preds" in metrics:
												_a								= "\n".join(metrics["preds"]					)
												generations_file.open("w+"					).write(__a					)




				@rank_zero_only
				def   UpperCamelCase__   (  self     :					int					,				__a     :					List[Any]					,				__a     :					Union[str, Any]					):
								try:
												_a								= pl_module.model.model.num_parameters()
								except AttributeError:
												_a								= pl_module.model.num_parameters()

								_a								= count_trainable_parameters(__a					)
								# mp stands for million parameters
								trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6}					)




				@rank_zero_only
				def   UpperCamelCase__   (  self     :					Union[str, Any]					,				__a     :					pl.Trainer					,				__a     :					pl.LightningModule					):
								save_json(pl_module.metrics					,				pl_module.metrics_save_path					)
								return self._write_logs(__a					,				__a					,				"test"					)




				@rank_zero_only
				def   UpperCamelCase__   (  self     :					Any					,				__a     :					pl.Trainer					,				__a     :					int					):
								save_json(pl_module.metrics					,				pl_module.metrics_save_path					)
								# Uncommenting this will save val generations
								# return self._write_logs(trainer, pl_module, "valid")
