






'''simple docstring'''

import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


A:   Union[str, Any]						=					logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

A:   Optional[Any]						=					{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
A:   Union[str, Any]						=					{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
A:   Optional[Any]						=					sorted(arg_to_scheduler.keys())
A:   Tuple						=					"{" + ", ".join(arg_to_scheduler_choices) + "}"






class 				__magic_name__  (						pl.LightningModule					):







	"""simple docstring"""



	def __init__(				self	,   _lowercase	,   _lowercase=None	,   _lowercase="base"	,   _lowercase=None	,   _lowercase=None	,   _lowercase=None	,   **_lowercase	,   )    ->	Dict:
							super().__init__()
							# TODO: move to self.save_hyperparameters()
							# self.save_hyperparameters()
							# can also expand arguments into trainer signature for easier reading

							self.save_hyperparameters(_lowercase							)
							lowercase_   : Dict   	=      0
							lowercase_   : Union[str, Any]   	=      Path(self.hparams.output_dir							)
							lowercase_   : Tuple   	=      self.hparams.cache_dir if self.hparams.cache_dir else None
							if config is None:
													lowercase_   : Any   	=      AutoConfig.from_pretrained(
													    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path	,   **({'num_labels': num_labels} if num_labels is not None else {})	,   cache_dir=_lowercase	,   **_lowercase	,   )
							else:
													lowercase_   : PretrainedConfig   	=      config

							lowercase_   : List[str]   	=      ('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
							for p in extra_model_params:
													if getattr(self.hparams	,   _lowercase	,   _lowercase							):
																			assert hasattr(self.config	,   _lowercase							), f"model config doesn't have a `{p}` attribute"
																			setattr(self.config	,   _lowercase	,   getattr(self.hparams	,   _lowercase							)							)

							if tokenizer is None:
													lowercase_   : Optional[int]   	=      AutoTokenizer.from_pretrained(
													    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path	,   cache_dir=_lowercase	,   )
							else:
													lowercase_   : PreTrainedTokenizer   	=      tokenizer
							lowercase_   : List[Any]   	=      MODEL_MODES[mode]
							if model is None:
													lowercase_   : int   	=      self.model_type.from_pretrained(
													    self.hparams.model_name_or_path	,   from_tf=bool('.ckpt' in self.hparams.model_name_or_path							)	,   config=self.config	,   cache_dir=_lowercase	,   )
							else:
													lowercase_   : str   	=      model
	def 							lowerCamelCase__		(				self	,   *_lowercase	,   **_lowercase							)    ->	Dict:
							lowercase_   : Dict   	=      self.model_type.from_pretrained(*_lowercase	,   **_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Optional[int]   	=      arg_to_scheduler[self.hparams.lr_scheduler]
							lowercase_   : str   	=      get_schedule_func(
							    self.opt	,   num_warmup_steps=self.hparams.warmup_steps	,   num_training_steps=self.total_steps()							)
							lowercase_   : Optional[int]   	=      {'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
							return scheduler
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							lowercase_   : List[str]   	=      self.model
							lowercase_   : Optional[int]   	=      ['bias', 'LayerNorm.weight']
							lowercase_   : List[str]   	=      [
							    {
							        'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay							)],
							        'weight_decay': self.hparams.weight_decay,
							    },
							    {
							        'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay							)],
							        'weight_decay': 0.0,
							    },
							]
							if self.hparams.adafactor:
													lowercase_   : Any   	=      Adafactor(
													    _lowercase	,   lr=self.hparams.learning_rate	,   scale_parameter=_lowercase	,   relative_step=_lowercase							)

							else:
													lowercase_   : List[str]   	=      AdamW(
													    _lowercase	,   lr=self.hparams.learning_rate	,   eps=self.hparams.adam_epsilon							)
							lowercase_   : List[str]   	=      optimizer

							lowercase_   : List[Any]   	=      self.get_lr_scheduler()

							return [optimizer], [scheduler]
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase							)    ->	List[str]:
							return self.validation_step(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	List[str]:
							return self.validation_end(_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	int:
							lowercase_   : List[Any]   	=      max(1	,   self.hparams.gpus							)  # TODO: consider num_tpu_cores
							lowercase_   : Any   	=      self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
							return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	str:
							if mode == "test":
													lowercase_   : Any   	=      len(self.test_dataloader().dataset							)
							else:
													lowercase_   : str   	=      self.get_dataloader('train'	,   self.hparams.train_batch_size	,   shuffle=_lowercase							)
													lowercase_   : List[str]   	=      len(self.train_dataloader().dataset							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase	,   _lowercase = False							)    ->	Tuple:
							raise NotImplementedError('You must implement this for your task'							)
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							return self.train_loader
	def 							lowerCamelCase__		(				self							)    ->	str:
							return self.get_dataloader('dev'	,   self.hparams.eval_batch_size	,   shuffle=_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	int:
							return self.get_dataloader('test'	,   self.hparams.eval_batch_size	,   shuffle=_lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Dict:
							return os.path.join(
							    self.hparams.data_dir	,   'cached_{}_{}_{}'.format(
							        _lowercase	,   list(filter(_lowercase	,   self.hparams.model_name_or_path.split('/'							)							)							).pop()	,   str(self.hparams.max_seq_length							)	,   )	,   )
	@pl.utilities.rank_zero_only
	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	None:
							lowercase_   : List[Any]   	=      self.output_dir.joinpath('best_tfmr'							)
							lowercase_   : Any   	=      self.step_count
							self.model.save_pretrained(_lowercase							)
							self.tokenizer.save_pretrained(_lowercase							)







	@staticmethod
	def 							lowerCamelCase__		(				_lowercase	,   _lowercase							)    ->	int:
							parser.add_argument(
							    '--model_name_or_path'	,   default=_lowercase	,   type=_lowercase	,   required=_lowercase	,   help='Path to pretrained model or model identifier from huggingface.co/models'	,   )
							parser.add_argument(
							    '--config_name'	,   default=''	,   type=_lowercase	,   help='Pretrained config name or path if not the same as model_name'							)
							parser.add_argument(
							    '--tokenizer_name'	,   default=_lowercase	,   type=_lowercase	,   help='Pretrained tokenizer name or path if not the same as model_name'	,   )
							parser.add_argument(
							    '--cache_dir'	,   default=''	,   type=_lowercase	,   help='Where do you want to store the pre-trained models downloaded from huggingface.co'	,   )
							parser.add_argument(
							    '--encoder_layerdrop'	,   type=_lowercase	,   help='Encoder layer dropout probability (Optional). Goes into model.config'	,   )
							parser.add_argument(
							    '--decoder_layerdrop'	,   type=_lowercase	,   help='Decoder layer dropout probability (Optional). Goes into model.config'	,   )
							parser.add_argument(
							    '--dropout'	,   type=_lowercase	,   help='Dropout probability (Optional). Goes into model.config'	,   )
							parser.add_argument(
							    '--attention_dropout'	,   type=_lowercase	,   help='Attention dropout probability (Optional). Goes into model.config'	,   )
							parser.add_argument('--learning_rate'	,   default=5E-5	,   type=_lowercase	,   help='The initial learning rate for Adam.'							)
							parser.add_argument(
							    '--lr_scheduler'	,   default='linear'	,   choices=_lowercase	,   metavar=_lowercase	,   type=_lowercase	,   help='Learning rate scheduler'	,   )
							parser.add_argument('--weight_decay'	,   default=0.0	,   type=_lowercase	,   help='Weight decay if we apply some.'							)
							parser.add_argument('--adam_epsilon'	,   default=1E-8	,   type=_lowercase	,   help='Epsilon for Adam optimizer.'							)
							parser.add_argument('--warmup_steps'	,   default=0	,   type=_lowercase	,   help='Linear warmup over warmup_steps.'							)
							parser.add_argument('--num_workers'	,   default=4	,   type=_lowercase	,   help='kwarg passed to DataLoader'							)
							parser.add_argument('--num_train_epochs'	,   dest='max_epochs'	,   default=3	,   type=_lowercase							)
							parser.add_argument('--train_batch_size'	,   default=32	,   type=_lowercase							)
							parser.add_argument('--eval_batch_size'	,   default=32	,   type=_lowercase							)
							parser.add_argument('--adafactor'	,   action='store_true'							)






class 				__magic_name__  (						pl.Callback					):







	"""simple docstring"""



	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase							)    ->	Union[str, Any]:
							lowercase_   : List[str]   	=      trainer.lr_schedulers[0]['scheduler']
							lowercase_   : Tuple   	=      {f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()							)}
							pl_module.logger.log_metrics(_lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase							)    ->	str:
							rank_zero_info('***** Validation results *****'							)
							lowercase_   : Dict   	=      trainer.callback_metrics
							# Log results
							for key in sorted(_lowercase							):
													if key not in ["log", "progress_bar"]:
																			rank_zero_info('{} = {}\n'.format(_lowercase	,   str(metrics[key]							)							)							)







	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase							)    ->	Any:
							rank_zero_info('***** Test results *****'							)
							lowercase_   : Any   	=      trainer.callback_metrics
							# Log and save results to file
							lowercase_   : List[str]   	=      os.path.join(pl_module.hparams.output_dir	,   'test_results.txt'							)
							with open(_lowercase	,   'w'							) as writer:
													for key in sorted(_lowercase							):
																			if key not in ["log", "progress_bar"]:
																									rank_zero_info('{} = {}\n'.format(_lowercase	,   str(metrics[key]							)							)							)
																									writer.write('{} = {}\n'.format(_lowercase	,   str(metrics[key]							)							)							)






def 						_UpperCAmelCase     (	a		:						Union[str, Any]    ,      a		:						Dict				)      ->      None:





						"""simple docstring"""
						#  To allow all pl args uncomment the following line
						#  parser = pl.Trainer.add_argparse_args(parser)
						parser.add_argument(
						    '--output_dir'    ,      default=a    ,      type=a    ,      required=a    ,      help='The output directory where the model predictions and checkpoints will be written.'    ,      )
						parser.add_argument(
						    '--fp16'    ,      action='store_true'    ,      help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'    ,      )

						parser.add_argument(
						    '--fp16_opt_level'    ,      type=a    ,      default='O2'    ,      help=(
						        'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
						        'See details at https://nvidia.github.io/apex/amp.html'
						    )    ,      )
						parser.add_argument('--n_tpu_cores'    ,      dest='tpu_cores'    ,      type=a				)
						parser.add_argument('--max_grad_norm'    ,      dest='gradient_clip_val'    ,      default=1.0    ,      type=a    ,      help='Max gradient norm'				)
						parser.add_argument('--do_train'    ,      action='store_true'    ,      help='Whether to run training.'				)
						parser.add_argument('--do_predict'    ,      action='store_true'    ,      help='Whether to run predictions on the test set.'				)
						parser.add_argument(
						    '--gradient_accumulation_steps'    ,      dest='accumulate_grad_batches'    ,      type=a    ,      default=1    ,      help='Number of updates steps to accumulate before performing a backward/update pass.'    ,      )
						parser.add_argument('--seed'    ,      type=a    ,      default=4_2    ,      help='random seed for initialization'				)
						parser.add_argument(
						    '--data_dir'    ,      default=a    ,      type=a    ,      required=a    ,      help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'    ,      )






def 						_UpperCAmelCase     (	a		:						BaseTransformer    ,      a		:						argparse.Namespace    ,      a		:						List[Any]=None    ,      a		:						str=True    ,      a		:						Tuple=[]    ,      a		:						Tuple=None    ,      a		:						List[str]=None    ,      **a		:						Dict    ,      )      ->      Optional[Any]:





						"""simple docstring"""
						pl.seed_everything(args.seed				)

						# init model
						lowercase_   : int   	=      Path(model.hparams.output_dir				)
						odir.mkdir(exist_ok=a				)

						# add custom checkpoints
						if checkpoint_callback is None:
												lowercase_   : str   	=      pl.callbacks.ModelCheckpoint(
												    filepath=args.output_dir    ,      prefix='checkpoint'    ,      monitor='val_loss'    ,      mode='min'    ,      save_top_k=1				)
						if early_stopping_callback:
												extra_callbacks.append(a				)
						if logging_callback is None:
												lowercase_   : Optional[Any]   	=      LoggingCallback()

						lowercase_   : Optional[int]   	=      {}

						# TODO: remove with PyTorch 1.6 since pl uses native amp
						if args.fpaa:
												lowercase_   : Any   	=      1_6
												lowercase_   : Optional[int]   	=      args.fpaa_opt_level

						if args.gpus > 1:
												lowercase_   : Union[str, Any]   	=      'ddp'

						lowercase_   : Any   	=      args.accumulate_grad_batches
						lowercase_   : Optional[Any]   	=      extra_train_kwargs.get('accelerator'    ,      a				)
						lowercase_   : Any   	=      extra_train_kwargs.get('profiler'    ,      a				)

						lowercase_   : List[str]   	=      pl.Trainer.from_argparse_args(
						    a    ,      weights_summary=a    ,      callbacks=[logging_callback] + extra_callbacks    ,      logger=a    ,      checkpoint_callback=a    ,      **a    ,      )

						if args.do_train:
												trainer.fit(a				)

						return trainer








'''simple docstring'''

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 				__magic_name__  (						unittest.TestCase					):







	"""simple docstring"""



	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Optional[int]   	=      list(tokenizer_class.max_model_input_sizes.keys()							)
							for model_name in sa_models[:1]:
													lowercase_   : List[str]   	=      tokenizer_class.from_pretrained(_lowercase							)
													self.assertIsNotNone(_lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)

													for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_lowercase	,   _lowercase							)
																			lowercase_   : Optional[int]   	=      tokenizer.convert_tokens_to_ids(_lowercase							)
																			self.assertIsInstance(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase = None							)    ->	str:
							lowercase_   : int   	=      pickle.dumps(_lowercase							)
							self.assertIsNotNone(_lowercase							)

							lowercase_   : Any   	=      pickle.loads(_lowercase							)

							# Ensure is_fast is correctly restored
							self.assertEqual(be_restored.is_fast	,   be_original.is_fast							)

							# Ensure encodings are potentially correctly restored
							if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings							)
							else:
													self.assertIsNone(be_restored.encodings							)

							# Ensure the keys are the same
							for original_v, restored_v in zip(be_original.values()	,   be_restored.values()							):
													if equal_op:
																			self.assertTrue(equal_op(_lowercase	,   _lowercase							)							)
													else:
																			self.assertEqual(_lowercase	,   _lowercase							)
	@slow
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							self.check_tokenizer_from_pretrained(_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							self.assertEqual(TensorType('tf'							)	,   TensorType.TENSORFLOW							)
							self.assertEqual(TensorType('pt'							)	,   TensorType.PYTORCH							)
							self.assertEqual(TensorType('np'							)	,   TensorType.NUMPY							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							import numpy as np

							lowercase_   : Any   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							# Python no tensor
							with self.subTest('BatchEncoding (Python, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)
	@require_tf
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							import tensorflow as tf

							def tf_array_equals(_lowercase	,   _lowercase							):
													return tf.reduce_all(tf.equal(_lowercase	,   _lowercase							)							)

							lowercase_   : Optional[Any]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)

							with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)
	@require_torch
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							import torch

							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Optional[int]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Tuple   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('Python Tokenizer'							):
													self.assertFalse(tokenizer_p('Small example to_encode'							).is_fast							)

							with self.subTest('Rust Tokenizer'							):
													self.assertTrue(tokenizer_r('Small example to_encode'							).is_fast							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : Optional[Any]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)
							lowercase_   : Dict   	=      tokenizer_r(['Test', '\xad', 'test']	,   is_split_into_words=_lowercase							)

							self.assertEqual(encoded.word_to_tokens(0							)	,   TokenSpan(start=1	,   end=2							)							)
							self.assertEqual(encoded.word_to_tokens(1							)	,   _lowercase							)
							self.assertEqual(encoded.word_to_tokens(2							)	,   TokenSpan(start=2	,   end=3							)							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Any   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : Union[str, Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[int]   	=      batch.convert_to_tensors(tensor_type='np'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Union[str, Any]   	=      batch.convert_to_tensors(tensor_type='pt'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : List[Any]   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Tuple   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='tf'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_flax
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : int   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : List[Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Any   	=      [{'input_ids': np.array([0, 1, 2]							)}, {'input_ids': np.array([0, 1, 2, 3]							)}]
							lowercase_   : str   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : List[str]   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='np'							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							import torch

							lowercase_   : Union[str, Any]   	=      [{'input_ids': torch.tensor([0, 1, 2]							)}, {'input_ids': torch.tensor([0, 1, 2, 3]							)}]
							lowercase_   : Optional[int]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : str   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Tuple   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='pt'							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Any:
							import tensorflow as tf

							lowercase_   : Any   	=      [{'input_ids': tf.constant([0, 1, 2]							)}, {'input_ids': tf.constant([0, 1, 2, 3]							)}]
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Dict   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='tf'							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[int]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							PreTrainedTokenizerFast(tokenizer_object=_lowercase							)







	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	int:
							lowercase_   : Union[str, Any]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(_lowercase	,   'tokenizer.json'							)							)
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase	,   'tokenizer.json'							)							)
