





import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the checkpointing capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

__UpperCAmelCase			:  str										=				16
__UpperCAmelCase			:  Any										=				32







def      A__				(       SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__ = 16)  ->       Any:
  __snake_case:		List[Any]					    =		AutoTokenizer.from_pretrained("""bert-base-cased""")
  __snake_case:		Tuple					    =		load_dataset("""glue"""   ,   """mrpc""")

  def tokenize_function(SCREAMING_SNAKE_CASE__):
    # max_length=None => use the model max length (it's actually the default)
    __snake_case:		str					    =		tokenizer(examples["""sentence1"""]   ,   examples["""sentence2"""]   ,   truncation=SCREAMING_SNAKE_CASE__   ,   max_length=SCREAMING_SNAKE_CASE__)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __snake_case:		Dict					    =		datasets.map(
        SCREAMING_SNAKE_CASE__   ,   batched=SCREAMING_SNAKE_CASE__   ,   remove_columns=["""idx""", """sentence1""", """sentence2"""]   ,   )

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __snake_case:		List[str]					    =		tokenized_datasets.rename_column("""label"""   ,   """labels""")

  def collate_fn(SCREAMING_SNAKE_CASE__):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __snake_case:		Dict					    =		128 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __snake_case:		int					    =		16
    elif accelerator.mixed_precision != "no":
      __snake_case:		int					    =		8
    else:
      __snake_case:		int					    =		None

    return tokenizer.pad(
        SCREAMING_SNAKE_CASE__   ,   padding="""longest"""   ,   max_length=SCREAMING_SNAKE_CASE__   ,   pad_to_multiple_of=SCREAMING_SNAKE_CASE__   ,   return_tensors="""pt"""   ,   )

  # Instantiate dataloaders.
  __snake_case:		Union[str, Any]					    =		DataLoader(
      tokenized_datasets["""train"""]   ,   shuffle=SCREAMING_SNAKE_CASE__   ,   collate_fn=SCREAMING_SNAKE_CASE__   ,   batch_size=SCREAMING_SNAKE_CASE__)
  __snake_case:		Optional[Any]					    =		DataLoader(
      tokenized_datasets["""validation"""]   ,   shuffle=SCREAMING_SNAKE_CASE__   ,   collate_fn=SCREAMING_SNAKE_CASE__   ,   batch_size=SCREAMING_SNAKE_CASE__)

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      __UpperCAmelCase			:  Union[str, Any]										=				mocked_dataloaders  # noqa: F811







def      A__				(       SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)  ->       str:
  # For testing only
  if os.environ.get("""TESTING_MOCKED_DATALOADERS"""   ,   SCREAMING_SNAKE_CASE__) == "1":
    __snake_case:		Dict					    =		2
  # Initialize accelerator
  __snake_case:		List[str]					    =		Accelerator(cpu=args.cpu   ,   mixed_precision=args.mixed_precision)
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __snake_case:		List[str]					    =		config["""lr"""]
  __snake_case:		Tuple					    =		int(config["""num_epochs"""])
  __snake_case:		Any					    =		int(config["""seed"""])
  __snake_case:		List[Any]					    =		int(config["""batch_size"""])

  # New Code #
  # Parse out whether we are saving every epoch or after a certain number of batches
  if hasattr(args.checkpointing_steps   ,   """isdigit"""):
    if args.checkpointing_steps == "epoch":
      __snake_case:		int					    =		args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
      __snake_case:		List[Any]					    =		int(args.checkpointing_steps)
    else:
      raise ValueError(
          F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.''')
  else:
    __snake_case:		Dict					    =		None

  set_seed(SCREAMING_SNAKE_CASE__)

  __snake_case    ,  __snake_case:		str					    =		get_dataloaders(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)
  __snake_case:		List[Any]					    =		evaluate.load("""glue"""   ,   """mrpc""")

  # If the batch size is too big we use gradient accumulation
  __snake_case:		Union[str, Any]					    =		1
  if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __snake_case:		Dict					    =		batch_size // MAX_GPU_BATCH_SIZE
    __snake_case:		Dict					    =		MAX_GPU_BATCH_SIZE

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __snake_case:		Dict					    =		AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""   ,   return_dict=SCREAMING_SNAKE_CASE__)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __snake_case:		Union[str, Any]					    =		model.to(accelerator.device)

  # Instantiate optimizer
  __snake_case:		List[str]					    =		AdamW(params=model.parameters()   ,   lr=SCREAMING_SNAKE_CASE__)

  # Instantiate scheduler
  __snake_case:		str					    =		get_linear_schedule_with_warmup(
      optimizer=SCREAMING_SNAKE_CASE__   ,   num_warmup_steps=100   ,   num_training_steps=(len(SCREAMING_SNAKE_CASE__) * num_epochs) // gradient_accumulation_steps   ,   )

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __snake_case    ,  __snake_case    ,  __snake_case    ,  __snake_case    ,  __snake_case:		Optional[int]					    =		accelerator.prepare(
      SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)

  # New Code #
  # We need to keep track of how many total steps we have iterated over
  __snake_case:		Optional[Any]					    =		0
  # We also need to keep track of the stating epoch so files are named properly
  __snake_case:		Any					    =		0

  # We need to load the checkpoint back in before training here with `load_state`
  # The total number of epochs is adjusted based on where the state is being loaded from,
  # as we assume continuation of the same training script
  if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
      accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}''')
      accelerator.load_state(args.resume_from_checkpoint)
      __snake_case:		Optional[int]					    =		os.path.basename(args.resume_from_checkpoint)
    else:
      # Get the most recent checkpoint
      __snake_case:		str					    =		[f.name for f in os.scandir(os.getcwd()) if f.is_dir()]
      dirs.sort(key=os.path.getctime)
      __snake_case:		Any					    =		dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __snake_case:		Optional[Any]					    =		os.path.splitext(SCREAMING_SNAKE_CASE__)[0]

    if "epoch" in training_difference:
      __snake_case:		List[Any]					    =		int(training_difference.replace("""epoch_"""   ,   """""")) + 1
      __snake_case:		Any					    =		None
    else:
      __snake_case:		Dict					    =		int(training_difference.replace("""step_"""   ,   """"""))
      __snake_case:		List[Any]					    =		resume_step // len(SCREAMING_SNAKE_CASE__)
      resume_step -= starting_epoch * len(SCREAMING_SNAKE_CASE__)

    # Now we train the model
  for epoch in range(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__):
    model.train()
    # New Code #
    if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
      # We need to skip steps until we reach the resumed step
      __snake_case:		Dict					    =		accelerator.skip_first_batches(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)
      overall_step += resume_step
    else:
      # After the first iteration though, we need to go back to the original dataloader
      __snake_case:		str					    =		train_dataloader
    for step, batch in enumerate(SCREAMING_SNAKE_CASE__):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device)
      __snake_case:		Union[str, Any]					    =		model(**SCREAMING_SNAKE_CASE__)
      __snake_case:		List[Any]					    =		outputs.loss
      __snake_case:		int					    =		loss / gradient_accumulation_steps
      accelerator.backward(SCREAMING_SNAKE_CASE__)
      if step % gradient_accumulation_steps == 0:
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
      # New Code #
      overall_step += 1

      # New Code #
      # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
      # These are saved to folders named `step_{overall_step}`
      # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
      # If mixed precision was used, will also save a "scalar.bin" file
      if isinstance(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__):
        __snake_case:		List[Any]					    =		F'''step_{overall_step}'''
        if overall_step % checkpointing_steps == 0:
          if args.output_dir is not None:
            __snake_case:		Any					    =		os.path.join(args.output_dir   ,   SCREAMING_SNAKE_CASE__)
          accelerator.save_state(SCREAMING_SNAKE_CASE__)

    model.eval()
    for step, batch in enumerate(SCREAMING_SNAKE_CASE__):
      # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
      batch.to(accelerator.device)
      with torch.no_grad():
        __snake_case:		Union[str, Any]					    =		model(**SCREAMING_SNAKE_CASE__)
      __snake_case:		Optional[int]					    =		outputs.logits.argmax(dim=-1)
      __snake_case    ,  __snake_case:		Union[str, Any]					    =		accelerator.gather_for_metrics((predictions, batch["""labels"""]))
      metric.add_batch(
          predictions=SCREAMING_SNAKE_CASE__   ,   references=SCREAMING_SNAKE_CASE__   ,   )

    __snake_case:		List[Any]					    =		metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(F'''epoch {epoch}:'''   ,   SCREAMING_SNAKE_CASE__)

    # New Code #
    # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
    # These are saved to folders named `epoch_{epoch}`
    # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
    # If mixed precision was used, will also save a "scalar.bin" file
    if checkpointing_steps == "epoch":
      __snake_case:		List[str]					    =		F'''epoch_{epoch}'''
      if args.output_dir is not None:
        __snake_case:		Dict					    =		os.path.join(args.output_dir   ,   SCREAMING_SNAKE_CASE__)
      accelerator.save_state(SCREAMING_SNAKE_CASE__)







def      A__				(       )  ->       List[Any]:
  __snake_case:		Optional[Any]					    =		argparse.ArgumentParser(description="""Simple example of training script.""")
  parser.add_argument(
      """--mixed_precision"""   ,   type=SCREAMING_SNAKE_CASE__   ,   default=SCREAMING_SNAKE_CASE__   ,   choices=["""no""", """fp16""", """bf16""", """fp8"""]   ,   help="""Whether to use mixed precision. Choose"""
      """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
      """and an Nvidia Ampere GPU."""   ,   )
  parser.add_argument("""--cpu"""   ,   action="""store_true"""   ,   help="""If passed, will train on the CPU.""")
  parser.add_argument(
      """--checkpointing_steps"""   ,   type=SCREAMING_SNAKE_CASE__   ,   default=SCREAMING_SNAKE_CASE__   ,   help="""Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch."""   ,   )
  parser.add_argument(
      """--output_dir"""   ,   type=SCREAMING_SNAKE_CASE__   ,   default="""."""   ,   help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""   ,   )
  parser.add_argument(
      """--resume_from_checkpoint"""   ,   type=SCREAMING_SNAKE_CASE__   ,   default=SCREAMING_SNAKE_CASE__   ,   help="""If the training should continue from a checkpoint folder."""   ,   )
  __snake_case:		int					    =		parser.parse_args()
  __snake_case:		List[Any]					    =		{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
  training_function(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)


if __name__ == "__main__":
      main()













import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          YosoForMaskedLM,
          YosoForMultipleChoice,
          YosoForQuestionAnswering,
          YosoForSequenceClassification,
          YosoForTokenClassification,
          YosoModel,
      )
      from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST







class        __snake_case			:





       '''simple docstring'''







       def __init__(    self					:       Union[str, Any]				,							A					:       Optional[Any]				,							A					:       str=13				,							A					:       List[Any]=7				,							A					:       str=True				,							A					:       Optional[int]=True				,							A					:       Any=True				,							A					:       List[str]=True				,							A					:       str=99				,							A					:       str=32				,							A					:       Any=5				,							A					:       Any=4				,							A					:       str=37				,							A					:       List[Any]="gelu"				,							A					:       Union[str, Any]=0.1				,							A					:       str=0.1				,							A					:       List[Any]=512				,							A					:       Optional[int]=16				,							A					:       List[str]=2				,							A					:       Optional[int]=0.02				,							A					:       Optional[int]=3				,							A					:       int=4				,							A					:       List[Any]=None				,							):
         __snake_case:		Any					    =		parent
         __snake_case:		Optional[int]					    =		batch_size
         __snake_case:		Optional[int]					    =		seq_length
         __snake_case:		Any					    =		is_training
         __snake_case:		int					    =		use_input_mask
         __snake_case:		Tuple					    =		use_token_type_ids
         __snake_case:		Optional[Any]					    =		use_labels
         __snake_case:		Optional[Any]					    =		vocab_size
         __snake_case:		str					    =		hidden_size
         __snake_case:		int					    =		num_hidden_layers
         __snake_case:		Optional[int]					    =		num_attention_heads
         __snake_case:		Tuple					    =		intermediate_size
         __snake_case:		Optional[int]					    =		hidden_act
         __snake_case:		Union[str, Any]					    =		hidden_dropout_prob
         __snake_case:		Any					    =		attention_probs_dropout_prob
         __snake_case:		List[str]					    =		max_position_embeddings
         __snake_case:		Any					    =		type_vocab_size
         __snake_case:		Dict					    =		type_sequence_label_size
         __snake_case:		int					    =		initializer_range
         __snake_case:		Union[str, Any]					    =		num_labels
         __snake_case:		int					    =		num_choices
         __snake_case:		List[Any]					    =		scope




       def 				UpperCAmelCase__					(    self					:       List[Any]						):
         __snake_case:		Any					    =		ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size						)

         __snake_case:		Any					    =		None
         if self.use_input_mask:
           __snake_case:		str					    =		random_attention_mask([self.batch_size, self.seq_length]						)

         __snake_case:		Tuple					    =		None
         if self.use_token_type_ids:
           __snake_case:		Tuple					    =		ids_tensor([self.batch_size, self.seq_length]				,							self.type_vocab_size						)

         __snake_case:		int					    =		None
         __snake_case:		Union[str, Any]					    =		None
         __snake_case:		str					    =		None
         if self.use_labels:
           __snake_case:		str					    =		ids_tensor([self.batch_size]				,							self.type_sequence_label_size						)
           __snake_case:		int					    =		ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels						)
           __snake_case:		Tuple					    =		ids_tensor([self.batch_size]				,							self.num_choices						)

         __snake_case:		List[str]					    =		self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




       def 				UpperCAmelCase__					(    self					:       Union[str, Any]						):
         return YosoConfig(
             vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							hidden_act=self.hidden_act				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							is_decoder=A				,							initializer_range=self.initializer_range				,							)




       def 				UpperCAmelCase__					(    self					:       Any						):
         __snake_case:		str					    =		self.get_config()
         __snake_case:		Optional[Any]					    =		300
         return config




       def 				UpperCAmelCase__					(    self					:       Optional[Any]						):
         (
             (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  
         ):		List[str]					    =		self.prepare_config_and_inputs()

         __snake_case:		Optional[Any]					    =		True
         __snake_case:		Optional[Any]					    =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
         __snake_case:		Tuple					    =		ids_tensor([self.batch_size, self.seq_length]				,							vocab_size=2						)

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )




       def 				UpperCAmelCase__					(    self					:       Any				,							A					:       Dict				,							A					:       str				,							A					:       Tuple				,							A					:       Dict				,							A					:       List[str]				,							A					:       List[str]				,							A					:       Optional[int]						):
         __snake_case:		Optional[int]					    =		YosoModel(config=A						)
         model.to(A						)
         model.eval()
         __snake_case:		List[Any]					    =		model(A				,							attention_mask=A				,							token_type_ids=A						)
         __snake_case:		str					    =		model(A				,							token_type_ids=A						)
         __snake_case:		int					    =		model(A						)
         self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)						)




       def 				UpperCAmelCase__					(    self					:       Optional[int]				,							A					:       Dict				,							A					:       int				,							A					:       str				,							A					:       List[str]				,							A					:       int				,							A					:       Optional[int]				,							A					:       str				,							A					:       Optional[int]				,							A					:       str				,							):
         __snake_case:		Optional[Any]					    =		True
         __snake_case:		List[str]					    =		YosoModel(A						)
         model.to(A						)
         model.eval()
         __snake_case:		Optional[int]					    =		model(
             A				,							attention_mask=A				,							token_type_ids=A				,							encoder_hidden_states=A				,							encoder_attention_mask=A				,							)
         __snake_case:		Optional[int]					    =		model(
             A				,							attention_mask=A				,							token_type_ids=A				,							encoder_hidden_states=A				,							)
         __snake_case:		Optional[int]					    =		model(A				,							attention_mask=A				,							token_type_ids=A						)
         self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)						)




       def 				UpperCAmelCase__					(    self					:       Optional[Any]				,							A					:       str				,							A					:       Optional[int]				,							A					:       List[Any]				,							A					:       Optional[Any]				,							A					:       Dict				,							A					:       int				,							A					:       Optional[int]						):
         __snake_case:		List[str]					    =		YosoForMaskedLM(config=A						)
         model.to(A						)
         model.eval()
         __snake_case:		int					    =		model(A				,							attention_mask=A				,							token_type_ids=A				,							labels=A						)
         self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)						)




       def 				UpperCAmelCase__					(    self					:       int				,							A					:       Any				,							A					:       Union[str, Any]				,							A					:       Optional[int]				,							A					:       Any				,							A					:       int				,							A					:       Any				,							A					:       Optional[int]						):
         __snake_case:		int					    =		YosoForQuestionAnswering(config=A						)
         model.to(A						)
         model.eval()
         __snake_case:		Any					    =		model(
             A				,							attention_mask=A				,							token_type_ids=A				,							start_positions=A				,							end_positions=A				,							)
         self.parent.assertEqual(result.start_logits.shape				,							(self.batch_size, self.seq_length)						)
         self.parent.assertEqual(result.end_logits.shape				,							(self.batch_size, self.seq_length)						)




       def 				UpperCAmelCase__					(    self					:       str				,							A					:       Union[str, Any]				,							A					:       Tuple				,							A					:       Union[str, Any]				,							A					:       int				,							A					:       Optional[int]				,							A					:       int				,							A					:       List[Any]						):
         __snake_case:		Union[str, Any]					    =		self.num_labels
         __snake_case:		Union[str, Any]					    =		YosoForSequenceClassification(A						)
         model.to(A						)
         model.eval()
         __snake_case:		Optional[int]					    =		model(A				,							attention_mask=A				,							token_type_ids=A				,							labels=A						)
         self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_labels)						)




       def 				UpperCAmelCase__					(    self					:       Dict				,							A					:       Optional[int]				,							A					:       Dict				,							A					:       Any				,							A					:       Dict				,							A					:       str				,							A					:       int				,							A					:       Optional[int]						):
         __snake_case:		List[Any]					    =		self.num_labels
         __snake_case:		Union[str, Any]					    =		YosoForTokenClassification(config=A						)
         model.to(A						)
         model.eval()
         __snake_case:		List[Any]					    =		model(A				,							attention_mask=A				,							token_type_ids=A				,							labels=A						)
         self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.num_labels)						)




       def 				UpperCAmelCase__					(    self					:       List[str]				,							A					:       Union[str, Any]				,							A					:       Tuple				,							A					:       int				,							A					:       Dict				,							A					:       Optional[int]				,							A					:       str				,							A					:       Optional[Any]						):
         __snake_case:		List[Any]					    =		self.num_choices
         __snake_case:		List[Any]					    =		YosoForMultipleChoice(config=A						)
         model.to(A						)
         model.eval()
         __snake_case:		str					    =		input_ids.unsqueeze(1						).expand(-1				,							self.num_choices				,							-1						).contiguous()
         __snake_case:		Any					    =		token_type_ids.unsqueeze(1						).expand(-1				,							self.num_choices				,							-1						).contiguous()
         __snake_case:		Optional[int]					    =		input_mask.unsqueeze(1						).expand(-1				,							self.num_choices				,							-1						).contiguous()
         __snake_case:		Any					    =		model(
             A				,							attention_mask=A				,							token_type_ids=A				,							labels=A				,							)
         self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.num_choices)						)






       def 				UpperCAmelCase__					(    self					:       List[Any]						):
         __snake_case:		List[str]					    =		self.prepare_config_and_inputs()
         (
             (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  
         ):		int					    =		config_and_inputs
         __snake_case:		str					    =		{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
         return config, inputs_dict







@require_torch
class        __snake_case			(				__lowerCamelCase  ,     __lowerCamelCase  ,     unittest.TestCase     ):





       '''simple docstring'''







       lowerCAmelCase__ 						=	(
           (
               YosoModel,
               YosoForMaskedLM,
               YosoForMultipleChoice,
               YosoForQuestionAnswering,
               YosoForSequenceClassification,
               YosoForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       lowerCAmelCase__ 						=	False
       lowerCAmelCase__ 						=	False
       lowerCAmelCase__ 						=	False

       lowerCAmelCase__ 						=	()
       lowerCAmelCase__ 						=	(
           {
               """feature-extraction""": YosoModel,
               """fill-mask""": YosoForMaskedLM,
               """question-answering""": YosoForQuestionAnswering,
               """text-classification""": YosoForSequenceClassification,
               """token-classification""": YosoForTokenClassification,
               """zero-shot""": YosoForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )




       def 				UpperCAmelCase__					(    self					:       str						):
         __snake_case:		List[str]					    =		YosoModelTester(self						)
         __snake_case:		Optional[int]					    =		ConfigTester(self				,							config_class=A				,							hidden_size=37						)




       def 				UpperCAmelCase__					(    self					:       List[str]						):
         self.config_tester.run_common_tests()




       def 				UpperCAmelCase__					(    self					:       Optional[int]						):
         __snake_case:		Union[str, Any]					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*A						)




       def 				UpperCAmelCase__					(    self					:       Union[str, Any]						):
         __snake_case:		Optional[Any]					    =		self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
           __snake_case:		Optional[Any]					    =		type
           self.model_tester.create_and_check_model(*A						)




       def 				UpperCAmelCase__					(    self					:       str						):
         __snake_case:		List[Any]					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*A						)




       def 				UpperCAmelCase__					(    self					:       List[Any]						):
         __snake_case:		List[str]					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*A						)




       def 				UpperCAmelCase__					(    self					:       int						):
         __snake_case:		Any					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*A						)




       def 				UpperCAmelCase__					(    self					:       Optional[Any]						):
         __snake_case:		List[Any]					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*A						)




       def 				UpperCAmelCase__					(    self					:       List[str]						):
         __snake_case:		Optional[int]					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*A						)




       @slow
       def 				UpperCAmelCase__					(    self					:       Any						):
         for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           __snake_case:		str					    =		YosoModel.from_pretrained(A						)
           self.assertIsNotNone(A						)






       def 				UpperCAmelCase__					(    self					:       Tuple						):
         return







@require_torch
class        __snake_case			(				unittest.TestCase     ):





       '''simple docstring'''







       @slow
       def 				UpperCAmelCase__					(    self					:       Tuple						):
         __snake_case:		Any					    =		YosoModel.from_pretrained("""uw-madison/yoso-4096"""						)
         __snake_case:		Union[str, Any]					    =		torch.tensor([[0, 1, 2, 3, 4, 5]]						)

         with torch.no_grad():
           __snake_case:		Optional[Any]					    =		model(A						)[0]

         __snake_case:		List[str]					    =		torch.Size((1, 6, 768)						)
         self.assertEqual(output.shape				,							A						)

         __snake_case:		List[Any]					    =		torch.tensor(
             [[[-0.0611, 0.1242, 0.0840], [0.0280, -0.0048, 0.1125], [0.0106, 0.0226, 0.0751]]]						)

         self.assertTrue(torch.allclose(output[:, :3, :3]				,							A				,							atol=1E-4						)						)




       @slow
       def 				UpperCAmelCase__					(    self					:       Optional[Any]						):
         __snake_case:		Optional[Any]					    =		YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""						)
         __snake_case:		Tuple					    =		torch.tensor([[0, 1, 2, 3, 4, 5]]						)

         with torch.no_grad():
           __snake_case:		Optional[int]					    =		model(A						)[0]

         __snake_case:		Tuple					    =		50_265

         __snake_case:		List[str]					    =		torch.Size((1, 6, vocab_size)						)
         self.assertEqual(output.shape				,							A						)

         __snake_case:		Any					    =		torch.tensor(
             [[[-2.1313, -3.7285, -2.2407], [-2.7047, -3.3314, -2.6408], [0.0629, -2.5166, -0.3356]]]						)

         self.assertTrue(torch.allclose(output[:, :3, :3]				,							A				,							atol=1E-4						)						)






       @slow
       def 				UpperCAmelCase__					(    self					:       Any						):
         __snake_case:		int					    =		YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""						)
         __snake_case:		Optional[Any]					    =		torch.arange(4_096						).unsqueeze(0						)

         with torch.no_grad():
           __snake_case:		Tuple					    =		model(A						)[0]

         __snake_case:		List[str]					    =		50_265

         __snake_case:		Union[str, Any]					    =		torch.Size((1, 4_096, vocab_size)						)
         self.assertEqual(output.shape				,							A						)

         __snake_case:		List[str]					    =		torch.tensor(
             [[[-2.3914, -4.3742, -5.0956], [-4.0988, -4.2384, -7.0406], [-3.1427, -3.7192, -6.6800]]]						)

         self.assertTrue(torch.allclose(output[:, :3, :3]				,							A				,							atol=1E-4						)						)






