

from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin







@dataclass
class 		__magic_name__     (				UpperCAmelCase_				):

   '''simple docstring'''




   lowerCamelCase__					:      torch.FloatTensor







class 		__magic_name__     (				UpperCAmelCase_						,			UpperCAmelCase_				):

   '''simple docstring'''




   @register_to_config
   def __init__(							self,       lowercase_ = 16,       lowercase_ = 88,       lowercase_ = None,       lowercase_ = None,       lowercase_ = 1,       lowercase_ = 0.0,       lowercase_ = 32,       lowercase_ = None,       lowercase_ = False,       lowercase_ = None,       lowercase_ = None,       lowercase_ = None,       lowercase_ = "geglu",       lowercase_ = None,       lowercase_ = False,       lowercase_ = False,       lowercase_ = False,       lowercase_ = "layer_norm",       lowercase_ = True,       )				->       Tuple:






     """simple docstring"""






     super().__init__()
     a__    =use_linear_projection
     a__    =num_attention_heads
     a__    =attention_head_dim
     a__    =num_attention_heads * attention_head_dim

     # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
     # Define whether input is continuous or discrete depending on configuration
     a__    =(in_channels is not None) and (patch_size is None)
     a__    =num_vector_embeds is not None
     a__    =in_channels is not None and patch_size is not None

     if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
       a__    =(
           F"""The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"""
           ''' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'''
           ''' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'''
           ''' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'''
           ''' would be very nice if you could open a Pull request for the `transformer/config.json` file'''
       )
       deprecate('''norm_type!=num_embeds_ada_norm''',       '''1.0.0''',       SCREAMING_SNAKE_CASE_,       standard_warn=SCREAMING_SNAKE_CASE_			)
       a__    ='''ada_norm'''

     if self.is_input_continuous and self.is_input_vectorized:
       raise ValueError(
           F"""Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"""
           ''' sure that either `in_channels` or `num_vector_embeds` is None.'''			)
     elif self.is_input_vectorized and self.is_input_patches:
       raise ValueError(
           F"""Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"""
           ''' sure that either `num_vector_embeds` or `num_patches` is None.'''			)
     elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
       raise ValueError(
           F"""Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"""
           F""" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."""			)

     # 2. Define input layers
     if self.is_input_continuous:
       a__    =in_channels

       a__    =torch.nn.GroupNorm(num_groups=SCREAMING_SNAKE_CASE_,       num_channels=SCREAMING_SNAKE_CASE_,       eps=1E-6,       affine=SCREAMING_SNAKE_CASE_			)
       if use_linear_projection:
         a__    =nn.Linear(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_			)
       else:
         a__    =nn.Convad(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       kernel_size=1,       stride=1,       padding=0			)
     elif self.is_input_vectorized:
       assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
       assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

       a__    =sample_size
       a__    =sample_size
       a__    =num_vector_embeds
       a__    =self.height * self.width

       a__    =ImagePositionalEmbeddings(
           num_embed=SCREAMING_SNAKE_CASE_,       embed_dim=SCREAMING_SNAKE_CASE_,       height=self.height,       width=self.width			)
     elif self.is_input_patches:
       assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

       a__    =sample_size
       a__    =sample_size

       a__    =patch_size
       a__    =PatchEmbed(
           height=SCREAMING_SNAKE_CASE_,       width=SCREAMING_SNAKE_CASE_,       patch_size=SCREAMING_SNAKE_CASE_,       in_channels=SCREAMING_SNAKE_CASE_,       embed_dim=SCREAMING_SNAKE_CASE_,       )

     # 3. Define transformers blocks
     a__    =nn.ModuleList(
         [
             BasicTransformerBlock(
                 SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       dropout=SCREAMING_SNAKE_CASE_,       cross_attention_dim=SCREAMING_SNAKE_CASE_,       activation_fn=SCREAMING_SNAKE_CASE_,       num_embeds_ada_norm=SCREAMING_SNAKE_CASE_,       attention_bias=SCREAMING_SNAKE_CASE_,       only_cross_attention=SCREAMING_SNAKE_CASE_,       upcast_attention=SCREAMING_SNAKE_CASE_,       norm_type=SCREAMING_SNAKE_CASE_,       norm_elementwise_affine=SCREAMING_SNAKE_CASE_,       )
             for d in range(SCREAMING_SNAKE_CASE_			)
         ]			)

     # 4. Define output layers
     a__    =in_channels if out_channels is None else out_channels
     if self.is_input_continuous:
       # TODO: should use out_channels for continuous projections
       if use_linear_projection:
         a__    =nn.Linear(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_			)
       else:
         a__    =nn.Convad(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       kernel_size=1,       stride=1,       padding=0			)
     elif self.is_input_vectorized:
       a__    =nn.LayerNorm(SCREAMING_SNAKE_CASE_			)
       a__    =nn.Linear(SCREAMING_SNAKE_CASE_,       self.num_vector_embeds - 1			)
     elif self.is_input_patches:
       a__    =nn.LayerNorm(SCREAMING_SNAKE_CASE_,       elementwise_affine=SCREAMING_SNAKE_CASE_,       eps=1E-6			)
       a__    =nn.Linear(SCREAMING_SNAKE_CASE_,       2 * inner_dim			)
       a__    =nn.Linear(SCREAMING_SNAKE_CASE_,       patch_size * patch_size * self.out_channels			)



   def 					_UpperCAmelCase					(							self,       lowercase_,       lowercase_ = None,       lowercase_ = None,       lowercase_ = None,       lowercase_ = None,       lowercase_ = None,       lowercase_ = None,       lowercase_ = True,       )				->       int:






     """simple docstring"""






     if attention_mask is not None and attention_mask.ndim == 2:
       # assume that mask is expressed as:
       #   (1 = keep,      0 = discard)
       # convert mask into a bias that can be added to attention scores:
       #       (keep = +0,     discard = -10000.0)
       a__    =(1 - attention_mask.to(hidden_states.dtype			)) * -10000.0
       a__    =attention_mask.unsqueeze(1			)

     # convert encoder_attention_mask to a bias the same way we do for attention_mask
     if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
       a__    =(1 - encoder_attention_mask.to(hidden_states.dtype			)) * -10000.0
       a__    =encoder_attention_mask.unsqueeze(1			)

     # 1. Input
     if self.is_input_continuous:
       a__    =hidden_states.shape
       a__    =hidden_states

       a__    =self.norm(SCREAMING_SNAKE_CASE_			)
       if not self.use_linear_projection:
         a__    =self.proj_in(SCREAMING_SNAKE_CASE_			)
         a__    =hidden_states.shape[1]
         a__    =hidden_states.permute(0,       2,       3,       1			).reshape(SCREAMING_SNAKE_CASE_,       height * width,       SCREAMING_SNAKE_CASE_			)
       else:
         a__    =hidden_states.shape[1]
         a__    =hidden_states.permute(0,       2,       3,       1			).reshape(SCREAMING_SNAKE_CASE_,       height * width,       SCREAMING_SNAKE_CASE_			)
         a__    =self.proj_in(SCREAMING_SNAKE_CASE_			)
     elif self.is_input_vectorized:
       a__    =self.latent_image_embedding(SCREAMING_SNAKE_CASE_			)
     elif self.is_input_patches:
       a__    =self.pos_embed(SCREAMING_SNAKE_CASE_			)

     # 2. Blocks
     for block in self.transformer_blocks:
       a__    =block(
           SCREAMING_SNAKE_CASE_,       attention_mask=SCREAMING_SNAKE_CASE_,       encoder_hidden_states=SCREAMING_SNAKE_CASE_,       encoder_attention_mask=SCREAMING_SNAKE_CASE_,       timestep=SCREAMING_SNAKE_CASE_,       cross_attention_kwargs=SCREAMING_SNAKE_CASE_,       class_labels=SCREAMING_SNAKE_CASE_,       )

     # 3. Output
     if self.is_input_continuous:
       if not self.use_linear_projection:
         a__    =hidden_states.reshape(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_			).permute(0,       3,       1,       2			).contiguous()
         a__    =self.proj_out(SCREAMING_SNAKE_CASE_			)
       else:
         a__    =self.proj_out(SCREAMING_SNAKE_CASE_			)
         a__    =hidden_states.reshape(SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_			).permute(0,       3,       1,       2			).contiguous()

       a__    =hidden_states + residual
     elif self.is_input_vectorized:
       a__    =self.norm_out(SCREAMING_SNAKE_CASE_			)
       a__    =self.out(SCREAMING_SNAKE_CASE_			)
       # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
       a__    =logits.permute(0,       2,       1			)

       # log(p(x_0))
       a__    =F.log_softmax(logits.double(),       dim=1			).float()
     elif self.is_input_patches:
       # TODO: cleanup!
       a__    =self.transformer_blocks[0].norma.emb(
           SCREAMING_SNAKE_CASE_,       SCREAMING_SNAKE_CASE_,       hidden_dtype=hidden_states.dtype			)
       a__    =self.proj_out_a(F.silu(SCREAMING_SNAKE_CASE_			)			).chunk(2,       dim=1			)
       a__    =self.norm_out(SCREAMING_SNAKE_CASE_			) * (1 + scale[:, None]) + shift[:, None]
       a__    =self.proj_out_a(SCREAMING_SNAKE_CASE_			)

       # unpatchify
       a__    =int(hidden_states.shape[1] ** 0.5			)
       a__    =hidden_states.reshape(
           shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)			)
       a__    =torch.einsum('''nhwpqc->nchpwq''',       SCREAMING_SNAKE_CASE_			)
       a__    =hidden_states.reshape(
           shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)			)

     if not return_dict:
       return (output,)

     return TransformeraDModelOutput(sample=SCREAMING_SNAKE_CASE_			)






'''simple docstring'''


import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         YosoForMaskedLM,
         YosoForMultipleChoice,
         YosoForQuestionAnswering,
         YosoForSequenceClassification,
         YosoForTokenClassification,
         YosoModel,
     )
     from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST






class 						lowerCAmelCase_	:







  '''simple docstring'''

  def __init__( self				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							int=13					,  SCREAMING_SNAKE_CASE_				:							List[str]=7					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=True					,  SCREAMING_SNAKE_CASE_				:							Tuple=True					,  SCREAMING_SNAKE_CASE_				:							Optional[int]=True					,  SCREAMING_SNAKE_CASE_				:							Optional[int]=True					,  SCREAMING_SNAKE_CASE_				:							Dict=99					,  SCREAMING_SNAKE_CASE_				:							Tuple=32					,  SCREAMING_SNAKE_CASE_				:							List[Any]=5					,  SCREAMING_SNAKE_CASE_				:							List[Any]=4					,  SCREAMING_SNAKE_CASE_				:							Dict=37					,  SCREAMING_SNAKE_CASE_				:							str="gelu"					,  SCREAMING_SNAKE_CASE_				:							str=0.1					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]=0.1					,  SCREAMING_SNAKE_CASE_				:							Dict=5_12					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=16					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=2					,  SCREAMING_SNAKE_CASE_				:							Any=0.02					,  SCREAMING_SNAKE_CASE_				:							Dict=3					,  SCREAMING_SNAKE_CASE_				:							Any=4					,  SCREAMING_SNAKE_CASE_				:							List[str]=None					,  )			->		Optional[int]:







       '''simple docstring'''



       A:		int          =		parent
       A:		Any          =		batch_size
       A:		List[Any]          =		seq_length
       A:		Union[str, Any]          =		is_training
       A:		List[str]          =		use_input_mask
       A:		List[str]          =		use_token_type_ids
       A:		Union[str, Any]          =		use_labels
       A:		str          =		vocab_size
       A:		Union[str, Any]          =		hidden_size
       A:		Tuple          =		num_hidden_layers
       A:		Dict          =		num_attention_heads
       A:		Any          =		intermediate_size
       A:		Tuple          =		hidden_act
       A:		Dict          =		hidden_dropout_prob
       A:		List[Any]          =		attention_probs_dropout_prob
       A:		int          =		max_position_embeddings
       A:		Any          =		type_vocab_size
       A:		int          =		type_sequence_label_size
       A:		str          =		initializer_range
       A:		Dict          =		num_labels
       A:		int          =		num_choices
       A:		str          =		scope



  def 					_snake_case     ( self				:							Optional[int]	)			->		List[Any]:







       '''simple docstring'''



       A:		Optional[int]          =		ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size	)

       A:		List[str]          =		None
       if self.use_input_mask:
            A:		Union[str, Any]          =		random_attention_mask([self.batch_size, self.seq_length]	)

       A:		Optional[int]          =		None
       if self.use_token_type_ids:
            A:		str          =		ids_tensor([self.batch_size, self.seq_length]					,  self.type_vocab_size	)

       A:		int          =		None
       A:		List[Any]          =		None
       A:		Dict          =		None
       if self.use_labels:
            A:		Dict          =		ids_tensor([self.batch_size]					,  self.type_sequence_label_size	)
            A:		Dict          =		ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels	)
            A:		int          =		ids_tensor([self.batch_size]					,  self.num_choices	)

       A:		List[str]          =		self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		Dict:







       '''simple docstring'''



       return YosoConfig(
           vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  type_vocab_size=self.type_vocab_size					,  is_decoder=SCREAMING_SNAKE_CASE_					,  initializer_range=self.initializer_range					,  )



  def 					_snake_case     ( self				:							Optional[Any]	)			->		Tuple:







       '''simple docstring'''



       A:		str          =		self.get_config()
       A:		Tuple          =		3_00
       return config



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		Optional[Any]:







       '''simple docstring'''



       (
           (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    
       ):		Optional[int]          =		self.prepare_config_and_inputs()

       A:		Any          =		True
       A:		Tuple          =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
       A:		int          =		ids_tensor([self.batch_size, self.seq_length]					,  vocab_size=2	)

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )



  def 					_snake_case     ( self				:							int					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]					,  SCREAMING_SNAKE_CASE_				:							int					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]	)			->		Optional[Any]:







       '''simple docstring'''



       A:		int          =		YosoModel(config=SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		str          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_	)
       A:		str          =		model(SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_	)
       A:		Any          =		model(SCREAMING_SNAKE_CASE_	)
       self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)	)



  def 					_snake_case     ( self				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							int					,  SCREAMING_SNAKE_CASE_				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							int					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							List[Any]					,  )			->		List[Any]:







       '''simple docstring'''



       A:		int          =		True
       A:		int          =		YosoModel(SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		Optional[int]          =		model(
           SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  encoder_hidden_states=SCREAMING_SNAKE_CASE_					,  encoder_attention_mask=SCREAMING_SNAKE_CASE_					,  )
       A:		Any          =		model(
           SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  encoder_hidden_states=SCREAMING_SNAKE_CASE_					,  )
       A:		List[str]          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_	)
       self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)	)



  def 					_snake_case     ( self				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Tuple					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Tuple	)			->		Union[str, Any]:







       '''simple docstring'''



       A:		List[Any]          =		YosoForMaskedLM(config=SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		Any          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  labels=SCREAMING_SNAKE_CASE_	)
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)	)



  def 					_snake_case     ( self				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							Tuple					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]	)			->		int:







       '''simple docstring'''



       A:		Optional[int]          =		YosoForQuestionAnswering(config=SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		Dict          =		model(
           SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  start_positions=SCREAMING_SNAKE_CASE_					,  end_positions=SCREAMING_SNAKE_CASE_					,  )
       self.parent.assertEqual(result.start_logits.shape					,  (self.batch_size, self.seq_length)	)
       self.parent.assertEqual(result.end_logits.shape					,  (self.batch_size, self.seq_length)	)



  def 					_snake_case     ( self				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							Tuple					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Tuple					,  SCREAMING_SNAKE_CASE_				:							Dict	)			->		List[Any]:







       '''simple docstring'''



       A:		str          =		self.num_labels
       A:		int          =		YosoForSequenceClassification(SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		Any          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  labels=SCREAMING_SNAKE_CASE_	)
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_labels)	)



  def 					_snake_case     ( self				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Dict					,  SCREAMING_SNAKE_CASE_				:							Dict	)			->		Optional[Any]:







       '''simple docstring'''



       A:		Tuple          =		self.num_labels
       A:		Optional[Any]          =		YosoForTokenClassification(config=SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		Dict          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  labels=SCREAMING_SNAKE_CASE_	)
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.num_labels)	)



  def 					_snake_case     ( self				:							Union[str, Any]					,  SCREAMING_SNAKE_CASE_				:							List[str]					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]	)			->		Any:







       '''simple docstring'''



       A:		Any          =		self.num_choices
       A:		Tuple          =		YosoForMultipleChoice(config=SCREAMING_SNAKE_CASE_	)
       model.to(SCREAMING_SNAKE_CASE_	)
       model.eval()
       A:		List[str]          =		input_ids.unsqueeze(1	).expand(-1					,  self.num_choices					,  -1	).contiguous()
       A:		Any          =		token_type_ids.unsqueeze(1	).expand(-1					,  self.num_choices					,  -1	).contiguous()
       A:		str          =		input_mask.unsqueeze(1	).expand(-1					,  self.num_choices					,  -1	).contiguous()
       A:		Union[str, Any]          =		model(
           SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  labels=SCREAMING_SNAKE_CASE_					,  )
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_choices)	)




  def 					_snake_case     ( self				:							int	)			->		Optional[int]:







       '''simple docstring'''



       A:		List[Any]          =		self.prepare_config_and_inputs()
       (
           (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    (
           A
       )     ,    
       ):		Optional[int]          =		config_and_inputs
       A:		Any          =		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict






@require_torch
class 						lowerCAmelCase_	(       UpperCAmelCase_       ,					UpperCAmelCase_       ,					unittest.TestCase							):







  '''simple docstring'''

  UpperCamelCase_			:      List[str]										=				(
      (
          YosoModel,
          YosoForMaskedLM,
          YosoForMultipleChoice,
          YosoForQuestionAnswering,
          YosoForSequenceClassification,
          YosoForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  UpperCamelCase_			:      Tuple										=				False
  UpperCamelCase_			:      Tuple										=				False
  UpperCamelCase_			:      List[str]										=				False

  UpperCamelCase_			:      List[str]										=				()
  UpperCamelCase_			:      Optional[int]										=				(
      {
          """feature-extraction""": YosoModel,
          """fill-mask""": YosoForMaskedLM,
          """question-answering""": YosoForQuestionAnswering,
          """text-classification""": YosoForSequenceClassification,
          """token-classification""": YosoForTokenClassification,
          """zero-shot""": YosoForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )



  def 					_snake_case     ( self				:							Dict	)			->		Any:







       '''simple docstring'''



       A:		Tuple          =		YosoModelTester(self	)
       A:		str          =		ConfigTester(self					,  config_class=SCREAMING_SNAKE_CASE_					,  hidden_size=37	)



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		Union[str, Any]:







       '''simple docstring'''



       self.config_tester.run_common_tests()



  def 					_snake_case     ( self				:							str	)			->		Optional[Any]:







       '''simple docstring'''



       A:		int          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		int:







       '''simple docstring'''



       A:		Any          =		self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
            A:		Tuple          =		type
            self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Tuple	)			->		Optional[int]:







       '''simple docstring'''



       A:		List[str]          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							List[Any]	)			->		str:







       '''simple docstring'''



       A:		str          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							List[str]	)			->		str:







       '''simple docstring'''



       A:		Tuple          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Tuple	)			->		Dict:







       '''simple docstring'''



       A:		int          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Dict	)			->		Dict:







       '''simple docstring'''



       A:		Dict          =		self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_	)



  @slow
  def 					_snake_case     ( self				:							int	)			->		int:







       '''simple docstring'''



       for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            A:		Dict          =		YosoModel.from_pretrained(SCREAMING_SNAKE_CASE_	)
            self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)




  def 					_snake_case     ( self				:							Optional[Any]	)			->		Any:







       '''simple docstring'''



       return






@require_torch
class 						lowerCAmelCase_	(       unittest.TestCase							):







  '''simple docstring'''

  @slow
  def 					_snake_case     ( self				:							Union[str, Any]	)			->		Union[str, Any]:







       '''simple docstring'''



       A:		List[Any]          =		YosoModel.from_pretrained('''uw-madison/yoso-4096'''	)
       A:		Dict          =		torch.tensor([[0, 1, 2, 3, 4, 5]]	)

       with torch.no_grad():
            A:		Union[str, Any]          =		model(SCREAMING_SNAKE_CASE_	)[0]

       A:		List[str]          =		torch.Size((1, 6, 7_68)	)
       self.assertEqual(output.shape					,  SCREAMING_SNAKE_CASE_	)

       A:		int          =		torch.tensor(
           [[[-0.0611, 0.1242, 0.0840], [0.0280, -0.0048, 0.1125], [0.0106, 0.0226, 0.0751]]]	)

       self.assertTrue(torch.allclose(output[:, :3, :3]					,  SCREAMING_SNAKE_CASE_					,  atol=1E-4	)	)



  @slow
  def 					_snake_case     ( self				:							int	)			->		Tuple:







       '''simple docstring'''



       A:		List[str]          =		YosoForMaskedLM.from_pretrained('''uw-madison/yoso-4096'''	)
       A:		Optional[int]          =		torch.tensor([[0, 1, 2, 3, 4, 5]]	)

       with torch.no_grad():
            A:		str          =		model(SCREAMING_SNAKE_CASE_	)[0]

       A:		Optional[int]          =		5_02_65

       A:		List[str]          =		torch.Size((1, 6, vocab_size)	)
       self.assertEqual(output.shape					,  SCREAMING_SNAKE_CASE_	)

       A:		Dict          =		torch.tensor(
           [[[-2.1313, -3.7285, -2.2407], [-2.7047, -3.3314, -2.6408], [0.0629, -2.5166, -0.3356]]]	)

       self.assertTrue(torch.allclose(output[:, :3, :3]					,  SCREAMING_SNAKE_CASE_					,  atol=1E-4	)	)




  @slow
  def 					_snake_case     ( self				:							Dict	)			->		Tuple:







       '''simple docstring'''



       A:		List[str]          =		YosoForMaskedLM.from_pretrained('''uw-madison/yoso-4096'''	)
       A:		Optional[int]          =		torch.arange(40_96	).unsqueeze(0	)

       with torch.no_grad():
            A:		int          =		model(SCREAMING_SNAKE_CASE_	)[0]

       A:		Optional[int]          =		5_02_65

       A:		Dict          =		torch.Size((1, 40_96, vocab_size)	)
       self.assertEqual(output.shape					,  SCREAMING_SNAKE_CASE_	)

       A:		Optional[int]          =		torch.tensor(
           [[[-2.3914, -4.3742, -5.0956], [-4.0988, -4.2384, -7.0406], [-3.1427, -3.7192, -6.6800]]]	)

       self.assertTrue(torch.allclose(output[:, :3, :3]					,  SCREAMING_SNAKE_CASE_					,  atol=1E-4	)	)






