



import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


lowerCAmelCase__		:     Union[str, Any]       						=logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met

class    UpperCAmelCase_  (   UpperCAmelCase_       ):

						'''simple docstring'''




						def __init__(    self      ,							*_A      ,							_A=None      ,							_A=None      ,							_A=None      ,							**_A		):







													'''simple docstring'''

													super().__init__(*__UpperCAmelCase      ,							**__UpperCAmelCase		)
													__SCREAMING_SNAKE_CASE									=					eval_examples
													__SCREAMING_SNAKE_CASE									=					post_process_function
													__SCREAMING_SNAKE_CASE									=					quant_trainer_args
													__SCREAMING_SNAKE_CASE									=					128  # default number of calibration samples






						def        _A	(    self      ,							_A=None		):







													'''simple docstring'''

													if calib_dataset is None and self.calib_dataset is None:
																				raise ValueError('Trainer: calibration requires an calib_dataset.'		)
													__SCREAMING_SNAKE_CASE									=					calib_dataset if calib_dataset is not None else self.calib_dataset

													__SCREAMING_SNAKE_CASE									=					self._remove_unused_columns(__UpperCAmelCase      ,							description='Calibration'		)

													return DataLoader(
													    __UpperCAmelCase      ,							batch_size=self.args.eval_batch_size      ,							collate_fn=self.data_collator      ,							drop_last=self.args.dataloader_drop_last      ,							num_workers=self.args.dataloader_num_workers      ,							pin_memory=self.args.dataloader_pin_memory      ,							shuffle=__UpperCAmelCase      ,							)






						def        _A	(    self      ,							_A=None		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									=					self.train_dataset if calib_dataset is None else calib_dataset
													__SCREAMING_SNAKE_CASE									=					self.get_calib_dataloader(__UpperCAmelCase		)

													__SCREAMING_SNAKE_CASE									=					self.model
													quant_trainer.configure_model(__UpperCAmelCase      ,							self.quant_trainer_args      ,							calib=__UpperCAmelCase		)
													model.eval()
													quant_trainer.enable_calibration(__UpperCAmelCase		)

													logger.info('***** Running calibration *****'		)
													logger.info(f"""  Num examples = {self.calib_num}"""		)
													logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""		)

													for step, inputs in enumerate(__UpperCAmelCase		):
																				# Prediction step
																				__SCREAMING_SNAKE_CASE							,       __SCREAMING_SNAKE_CASE							,       __SCREAMING_SNAKE_CASE									=					self.prediction_step(__UpperCAmelCase      ,							__UpperCAmelCase      ,							prediction_loss_only=__UpperCAmelCase		)
																				if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																											break

													quant_trainer.finish_calibration(__UpperCAmelCase      ,							self.quant_trainer_args		)
													__SCREAMING_SNAKE_CASE									=					model






						def        _A	(    self      ,							_A=None      ,							_A=None      ,							_A=None      ,							_A = "eval"		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									=					self.eval_dataset if eval_dataset is None else eval_dataset
													__SCREAMING_SNAKE_CASE									=					self.get_eval_dataloader(__UpperCAmelCase		)
													__SCREAMING_SNAKE_CASE									=					self.eval_examples if eval_examples is None else eval_examples

													# Temporarily disable metric computation, we will do it in the loop here.
													__SCREAMING_SNAKE_CASE									=					self.compute_metrics
													__SCREAMING_SNAKE_CASE									=					None
													__SCREAMING_SNAKE_CASE									=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
													try:
																				__SCREAMING_SNAKE_CASE									=					eval_loop(
																				    __UpperCAmelCase      ,							description='Evaluation'      ,							prediction_loss_only=True if compute_metrics is None else None      ,							ignore_keys=__UpperCAmelCase      ,							)
													finally:
																				__SCREAMING_SNAKE_CASE									=					compute_metrics

													if self.post_process_function is not None and self.compute_metrics is not None:
																				__SCREAMING_SNAKE_CASE									=					self.post_process_function(__UpperCAmelCase      ,							__UpperCAmelCase      ,							output.predictions		)
																				__SCREAMING_SNAKE_CASE									=					self.compute_metrics(__UpperCAmelCase		)

																				# Prefix all keys with metric_key_prefix + '_'
																				for key in list(metrics.keys()		):
																											if not key.startswith(f"""{metric_key_prefix}_"""		):
																																		__SCREAMING_SNAKE_CASE									=					metrics.pop(__UpperCAmelCase		)

																				self.log(__UpperCAmelCase		)
													else:
																				__SCREAMING_SNAKE_CASE									=					{}

													if self.args.tpu_metrics_debug or self.args.debug:
																				# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																				xm.master_print(met.metrics_report()		)

													__SCREAMING_SNAKE_CASE									=					self.callback_handler.on_evaluate(self.args      ,							self.state      ,							self.control      ,							__UpperCAmelCase		)
													return metrics






						def        _A	(    self      ,							_A      ,							_A      ,							_A=None      ,							_A = "test"		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									=					self.get_test_dataloader(__UpperCAmelCase		)

													# Temporarily disable metric computation, we will do it in the loop here.
													__SCREAMING_SNAKE_CASE									=					self.compute_metrics
													__SCREAMING_SNAKE_CASE									=					None
													__SCREAMING_SNAKE_CASE									=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
													try:
																				__SCREAMING_SNAKE_CASE									=					eval_loop(
																				    __UpperCAmelCase      ,							description='Prediction'      ,							prediction_loss_only=True if compute_metrics is None else None      ,							ignore_keys=__UpperCAmelCase      ,							)
													finally:
																				__SCREAMING_SNAKE_CASE									=					compute_metrics

													if self.post_process_function is None or self.compute_metrics is None:
																				return output

													__SCREAMING_SNAKE_CASE									=					self.post_process_function(__UpperCAmelCase      ,							__UpperCAmelCase      ,							output.predictions      ,							'predict'		)
													__SCREAMING_SNAKE_CASE									=					self.compute_metrics(__UpperCAmelCase		)

													# Prefix all keys with metric_key_prefix + '_'
													for key in list(metrics.keys()		):
																				if not key.startswith(f"""{metric_key_prefix}_"""		):
																											__SCREAMING_SNAKE_CASE									=					metrics.pop(__UpperCAmelCase		)

													return PredictionOutput(predictions=predictions.predictions      ,							label_ids=predictions.label_ids      ,							metrics=__UpperCAmelCase		)






						def        _A	(    self      ,							_A="./"		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									=					self.eval_dataset
													__SCREAMING_SNAKE_CASE									=					self.get_eval_dataloader(__UpperCAmelCase		)

													__SCREAMING_SNAKE_CASE									=					next(iter(__UpperCAmelCase		)		)

													# saving device - to make it consistent
													__SCREAMING_SNAKE_CASE									=					torch.device('cuda' if torch.cuda.is_available() else 'cpu'		)

													# convert to tuple
													__SCREAMING_SNAKE_CASE									=					tuple(v.to(__UpperCAmelCase		) for k, v in batch.items()		)

													logger.info('Converting model to be onnx compatible'		)
													from pytorch_quantization.nn import TensorQuantizer

													__SCREAMING_SNAKE_CASE									=					True

													__SCREAMING_SNAKE_CASE									=					self.model.to(__UpperCAmelCase		)

													model.eval()
													model.float()

													__SCREAMING_SNAKE_CASE									=					model.module if hasattr(__UpperCAmelCase      ,							'module'		) else model
													quant_trainer.configure_model(__UpperCAmelCase      ,							self.quant_trainer_args		)

													__SCREAMING_SNAKE_CASE									=					os.path.join(__UpperCAmelCase      ,							'model.onnx'		)
													logger.info(f"""exporting model to {output_model_file}"""		)

													__SCREAMING_SNAKE_CASE									=					{0: 'batch_size', 1: 'seq_len'}

													torch.onnx.export(
													    __UpperCAmelCase      ,							__UpperCAmelCase      ,							__UpperCAmelCase      ,							export_params=__UpperCAmelCase      ,							opset_version=13      ,							do_constant_folding=__UpperCAmelCase      ,							input_names=['input_ids', 'attention_mask', 'token_type_ids']      ,							output_names=['output_start_logits', 'output_end_logits']      ,							dynamic_axes={
													        'input_ids': axes,
													        'attention_mask': axes,
													        'token_type_ids': axes,
													        'output_start_logits': axes,
													        'output_end_logits': axes,
													    }      ,							verbose=__UpperCAmelCase      ,							)
													logger.info('onnx export finished'		)







import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCamelCase__											=					logging.getLogger(__name__)




def    lowerCAmelCase_ (     )     ->				int:







      '''simple docstring'''


      UpperCAmelCase__								=  argparse.ArgumentParser(
          description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."							)
      parser.add_argument("--file_path",  type=__A,  default="data/dump.txt",  help="The path to the data."							)
      parser.add_argument("--tokenizer_type",  type=__A,  default="bert",  choices=["bert", "roberta", "gpt2"]							)
      parser.add_argument("--tokenizer_name",  type=__A,  default="bert-base-uncased",  help="The tokenizer to use."							)
      parser.add_argument("--dump_file",  type=__A,  default="data/dump",  help="The dump file prefix."							)
      UpperCAmelCase__								=  parser.parse_args()

      logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""							)
      if args.tokenizer_type == "bert":
            UpperCAmelCase__								=  BertTokenizer.from_pretrained(args.tokenizer_name							)
            UpperCAmelCase__								=  tokenizer.special_tokens_map["cls_token"]  # `[CLS]`
            UpperCAmelCase__								=  tokenizer.special_tokens_map["sep_token"]  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            UpperCAmelCase__								=  RobertaTokenizer.from_pretrained(args.tokenizer_name							)
            UpperCAmelCase__								=  tokenizer.special_tokens_map["cls_token"]  # `<s>`
            UpperCAmelCase__								=  tokenizer.special_tokens_map["sep_token"]  # `</s>`
      elif args.tokenizer_type == "gpt2":
            UpperCAmelCase__								=  GPTaTokenizer.from_pretrained(args.tokenizer_name							)
            UpperCAmelCase__								=  tokenizer.special_tokens_map["bos_token"]  # `<|endoftext|>`
            UpperCAmelCase__								=  tokenizer.special_tokens_map["eos_token"]  # `<|endoftext|>`

      logger.info(f"""Loading text from {args.file_path}"""							)
      with open(args.file_path,  "r",  encoding="utf8"							) as fp:
            UpperCAmelCase__								=  fp.readlines()

      logger.info("Start encoding"							)
      logger.info(f"""{len(__A							)} examples to process."""							)

      UpperCAmelCase__								=  []
      UpperCAmelCase__								=  0
      UpperCAmelCase__								=  10_000
      UpperCAmelCase__								=  time.time()
      for text in data:
            UpperCAmelCase__								=  f"""{bos} {text.strip()} {sep}"""
            UpperCAmelCase__								=  tokenizer.encode(__A,  add_special_tokens=__A							)
            rslt.append(__A							)

            iter += 1
            if iter % interval == 0:
                  UpperCAmelCase__								=  time.time()
                  logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""							)
                  UpperCAmelCase__								=  time.time()
      logger.info("Finished binarization"							)
      logger.info(f"""{len(__A							)} examples processed."""							)

      UpperCAmelCase__								=  f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
      UpperCAmelCase__								=  tokenizer.vocab_size
      if vocab_size < (1 << 16):
            UpperCAmelCase__								=  [np.uintaa(__A							) for d in rslt]
      else:
            UpperCAmelCase__								=  [np.intaa(__A							) for d in rslt]
      random.shuffle(rslt_							)
      logger.info(f"""Dump to {dp_file}"""							)
      with open(__A,  "wb"							) as handle:
            pickle.dump(rslt_,  __A,  protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
      main()



