
from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin


@dataclass
class 			UpperCamelCase_				(  snake_case__     ):




      '''simple docstring'''







      a		:Union[str, Any]						 =      42


class 			UpperCamelCase_				(  snake_case__					,       snake_case__     ):




      '''simple docstring'''







      @register_to_config
      def __init__( self					,					_UpperCAmelCase = 16					,					_UpperCAmelCase = 88					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = 1					,					_UpperCAmelCase = 0.0					,					_UpperCAmelCase = 32					,					_UpperCAmelCase = None					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = "geglu"					,					_UpperCAmelCase = None					,					_UpperCAmelCase = False					,					_UpperCAmelCase = False					,					_UpperCAmelCase = False					,					_UpperCAmelCase = "layer_norm"					,					_UpperCAmelCase = True					,					):
            super().__init__()
            lowerCAmelCase_						=      use_linear_projection
            lowerCAmelCase_						=      num_attention_heads
            lowerCAmelCase_						=      attention_head_dim
            lowerCAmelCase_						=      num_attention_heads * attention_head_dim

            # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
            # Define whether input is continuous or discrete depending on configuration
            lowerCAmelCase_						=      (in_channels is not None) and (patch_size is None)
            lowerCAmelCase_						=      num_vector_embeds is not None
            lowerCAmelCase_						=      in_channels is not None and patch_size is not None

            if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
                  lowerCAmelCase_						=      (
                      f'The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'
                      " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
                      " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
                      " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
                      " would be very nice if you could open a Pull request for the `transformer/config.json` file"
                  )
                  deprecate('''norm_type!=num_embeds_ada_norm'''					,					'''1.0.0'''					,					__SCREAMING_SNAKE_CASE					,					standard_warn=__SCREAMING_SNAKE_CASE)
                  lowerCAmelCase_						=      "ada_norm"

            if self.is_input_continuous and self.is_input_vectorized:
                  raise ValueError(
                      f'Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'
                      ''' sure that either `in_channels` or `num_vector_embeds` is None.''')
            elif self.is_input_vectorized and self.is_input_patches:
                  raise ValueError(
                      f'Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'
                      ''' sure that either `num_vector_embeds` or `num_patches` is None.''')
            elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
                  raise ValueError(
                      f'Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'
                      f' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.')

            # 2. Define input layers
            if self.is_input_continuous:
                  lowerCAmelCase_						=      in_channels

                  lowerCAmelCase_						=      torch.nn.GroupNorm(num_groups=__SCREAMING_SNAKE_CASE					,					num_channels=__SCREAMING_SNAKE_CASE					,					eps=1E-6					,					affine=__SCREAMING_SNAKE_CASE)
                  if use_linear_projection:
                        lowerCAmelCase_						=      nn.Linear(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE)
                  else:
                        lowerCAmelCase_						=      nn.Convad(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					kernel_size=1					,					stride=1					,					padding=0)
            elif self.is_input_vectorized:
                  assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
                  assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

                  lowerCAmelCase_						=      sample_size
                  lowerCAmelCase_						=      sample_size
                  lowerCAmelCase_						=      num_vector_embeds
                  lowerCAmelCase_						=      self.height * self.width

                  lowerCAmelCase_						=      ImagePositionalEmbeddings(
                      num_embed=__SCREAMING_SNAKE_CASE					,					embed_dim=__SCREAMING_SNAKE_CASE					,					height=self.height					,					width=self.width)
            elif self.is_input_patches:
                  assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

                  lowerCAmelCase_						=      sample_size
                  lowerCAmelCase_						=      sample_size

                  lowerCAmelCase_						=      patch_size
                  lowerCAmelCase_						=      PatchEmbed(
                      height=__SCREAMING_SNAKE_CASE					,					width=__SCREAMING_SNAKE_CASE					,					patch_size=__SCREAMING_SNAKE_CASE					,					in_channels=__SCREAMING_SNAKE_CASE					,					embed_dim=__SCREAMING_SNAKE_CASE					,					)

            # 3. Define transformers blocks
            lowerCAmelCase_						=      nn.ModuleList(
                [
                    BasicTransformerBlock(
                        __SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					dropout=__SCREAMING_SNAKE_CASE					,					cross_attention_dim=__SCREAMING_SNAKE_CASE					,					activation_fn=__SCREAMING_SNAKE_CASE					,					num_embeds_ada_norm=__SCREAMING_SNAKE_CASE					,					attention_bias=__SCREAMING_SNAKE_CASE					,					only_cross_attention=__SCREAMING_SNAKE_CASE					,					upcast_attention=__SCREAMING_SNAKE_CASE					,					norm_type=__SCREAMING_SNAKE_CASE					,					norm_elementwise_affine=__SCREAMING_SNAKE_CASE					,					)
                    for d in range(__SCREAMING_SNAKE_CASE)
                ])

            # 4. Define output layers
            lowerCAmelCase_						=      in_channels if out_channels is None else out_channels
            if self.is_input_continuous:
                  # TODO: should use out_channels for continuous projections
                  if use_linear_projection:
                        lowerCAmelCase_						=      nn.Linear(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE)
                  else:
                        lowerCAmelCase_						=      nn.Convad(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					kernel_size=1					,					stride=1					,					padding=0)
            elif self.is_input_vectorized:
                  lowerCAmelCase_						=      nn.LayerNorm(__SCREAMING_SNAKE_CASE)
                  lowerCAmelCase_						=      nn.Linear(__SCREAMING_SNAKE_CASE					,					self.num_vector_embeds - 1)
            elif self.is_input_patches:
                  lowerCAmelCase_						=      nn.LayerNorm(__SCREAMING_SNAKE_CASE					,					elementwise_affine=__SCREAMING_SNAKE_CASE					,					eps=1E-6)
                  lowerCAmelCase_						=      nn.Linear(__SCREAMING_SNAKE_CASE					,					2 * inner_dim)
                  lowerCAmelCase_						=      nn.Linear(__SCREAMING_SNAKE_CASE					,					patch_size * patch_size * self.out_channels)







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = None					,					_UpperCAmelCase = True					,					):







            if attention_mask is not None and attention_mask.ndim == 2:
                  # assume that mask is expressed as:
                  #   (1 = keep,      0 = discard)
                  # convert mask into a bias that can be added to attention scores:
                  #       (keep = +0,     discard = -10000.0)
                  lowerCAmelCase_						=      (1 - attention_mask.to(hidden_states.dtype)) * -10_000.0
                  lowerCAmelCase_						=      attention_mask.unsqueeze(1)

            # convert encoder_attention_mask to a bias the same way we do for attention_mask
            if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
                  lowerCAmelCase_						=      (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10_000.0
                  lowerCAmelCase_						=      encoder_attention_mask.unsqueeze(1)

            # 1. Input
            if self.is_input_continuous:
                  lowerCAmelCase_						=      hidden_states.shape
                  lowerCAmelCase_						=      hidden_states

                  lowerCAmelCase_						=      self.norm(__SCREAMING_SNAKE_CASE)
                  if not self.use_linear_projection:
                        lowerCAmelCase_						=      self.proj_in(__SCREAMING_SNAKE_CASE)
                        lowerCAmelCase_						=      hidden_states.shape[1]
                        lowerCAmelCase_						=      hidden_states.permute(0					,					2					,					3					,					1).reshape(__SCREAMING_SNAKE_CASE					,					height * width					,					__SCREAMING_SNAKE_CASE)
                  else:
                        lowerCAmelCase_						=      hidden_states.shape[1]
                        lowerCAmelCase_						=      hidden_states.permute(0					,					2					,					3					,					1).reshape(__SCREAMING_SNAKE_CASE					,					height * width					,					__SCREAMING_SNAKE_CASE)
                        lowerCAmelCase_						=      self.proj_in(__SCREAMING_SNAKE_CASE)
            elif self.is_input_vectorized:
                  lowerCAmelCase_						=      self.latent_image_embedding(__SCREAMING_SNAKE_CASE)
            elif self.is_input_patches:
                  lowerCAmelCase_						=      self.pos_embed(__SCREAMING_SNAKE_CASE)

            # 2. Blocks
            for block in self.transformer_blocks:
                  lowerCAmelCase_						=      block(
                      __SCREAMING_SNAKE_CASE					,					attention_mask=__SCREAMING_SNAKE_CASE					,					encoder_hidden_states=__SCREAMING_SNAKE_CASE					,					encoder_attention_mask=__SCREAMING_SNAKE_CASE					,					timestep=__SCREAMING_SNAKE_CASE					,					cross_attention_kwargs=__SCREAMING_SNAKE_CASE					,					class_labels=__SCREAMING_SNAKE_CASE					,					)

            # 3. Output
            if self.is_input_continuous:
                  if not self.use_linear_projection:
                        lowerCAmelCase_						=      hidden_states.reshape(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE).permute(0					,					3					,					1					,					2).contiguous()
                        lowerCAmelCase_						=      self.proj_out(__SCREAMING_SNAKE_CASE)
                  else:
                        lowerCAmelCase_						=      self.proj_out(__SCREAMING_SNAKE_CASE)
                        lowerCAmelCase_						=      hidden_states.reshape(__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE).permute(0					,					3					,					1					,					2).contiguous()

                  lowerCAmelCase_						=      hidden_states + residual
            elif self.is_input_vectorized:
                  lowerCAmelCase_						=      self.norm_out(__SCREAMING_SNAKE_CASE)
                  lowerCAmelCase_						=      self.out(__SCREAMING_SNAKE_CASE)
                  # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
                  lowerCAmelCase_						=      logits.permute(0					,					2					,					1)

                  # log(p(x_0))
                  lowerCAmelCase_						=      F.log_softmax(logits.double()					,					dim=1).float()
            elif self.is_input_patches:
                  # TODO: cleanup!
                  lowerCAmelCase_						=      self.transformer_blocks[0].norma.emb(
                      __SCREAMING_SNAKE_CASE					,					__SCREAMING_SNAKE_CASE					,					hidden_dtype=hidden_states.dtype)
                  lowerCAmelCase_						=      self.proj_out_a(F.silu(__SCREAMING_SNAKE_CASE)).chunk(2					,					dim=1)
                  lowerCAmelCase_						=      self.norm_out(__SCREAMING_SNAKE_CASE) * (1 + scale[:, None]) + shift[:, None]
                  lowerCAmelCase_						=      self.proj_out_a(__SCREAMING_SNAKE_CASE)

                  # unpatchify
                  lowerCAmelCase_						=      int(hidden_states.shape[1] ** 0.5)
                  lowerCAmelCase_						=      hidden_states.reshape(
                      shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels))
                  lowerCAmelCase_						=      torch.einsum('''nhwpqc->nchpwq'''					,					__SCREAMING_SNAKE_CASE)
                  lowerCAmelCase_						=      hidden_states.reshape(
                      shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size))

            if not return_dict:
                  return (output,)

            return TransformeraDModelOutput(sample=__SCREAMING_SNAKE_CASE)







import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         ErnieMForInformationExtraction,
         ErnieMForMultipleChoice,
         ErnieMForQuestionAnswering,
         ErnieMForSequenceClassification,
         ErnieMForTokenClassification,
         ErnieMModel,
     )
     from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST



class 							lowercase__:






      '''simple docstring'''







      def __init__(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=13   ,			__SCREAMING_SNAKE_CASE=7   ,			__SCREAMING_SNAKE_CASE=True   ,			__SCREAMING_SNAKE_CASE=True   ,			__SCREAMING_SNAKE_CASE=True   ,			__SCREAMING_SNAKE_CASE=99   ,			__SCREAMING_SNAKE_CASE=32   ,			__SCREAMING_SNAKE_CASE=5   ,			__SCREAMING_SNAKE_CASE=4   ,			__SCREAMING_SNAKE_CASE=37   ,			__SCREAMING_SNAKE_CASE="gelu"   ,			__SCREAMING_SNAKE_CASE=0.1   ,			__SCREAMING_SNAKE_CASE=0.1   ,			__SCREAMING_SNAKE_CASE=5_12   ,			__SCREAMING_SNAKE_CASE=16   ,			__SCREAMING_SNAKE_CASE=2   ,			__SCREAMING_SNAKE_CASE=0.02   ,			__SCREAMING_SNAKE_CASE=3   ,			__SCREAMING_SNAKE_CASE=4   ,			__SCREAMING_SNAKE_CASE=None   ,			)					->						Any:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=parent
         UpperCamelCase__ :			Tuple 						=batch_size
         UpperCamelCase__ :			Union[str, Any] 						=seq_length
         UpperCamelCase__ :			Union[str, Any] 						=is_training
         UpperCamelCase__ :			Tuple 						=use_input_mask
         UpperCamelCase__ :			Union[str, Any] 						=use_labels
         UpperCamelCase__ :			Union[str, Any] 						=vocab_size
         UpperCamelCase__ :			List[Any] 						=hidden_size
         UpperCamelCase__ :			Dict 						=num_hidden_layers
         UpperCamelCase__ :			Any 						=num_attention_heads
         UpperCamelCase__ :			List[str] 						=intermediate_size
         UpperCamelCase__ :			List[Any] 						=hidden_act
         UpperCamelCase__ :			Tuple 						=hidden_dropout_prob
         UpperCamelCase__ :			Any 						=attention_probs_dropout_prob
         UpperCamelCase__ :			Tuple 						=max_position_embeddings
         UpperCamelCase__ :			Dict 						=type_vocab_size
         UpperCamelCase__ :			Tuple 						=type_sequence_label_size
         UpperCamelCase__ :			Tuple 						=initializer_range
         UpperCamelCase__ :			List[str] 						=num_labels
         UpperCamelCase__ :			Any 						=num_choices
         UpperCamelCase__ :			Tuple 						=scope




      def    UpperCAmelCase							(		self)					->						int:







         """simple docstring"""







         UpperCamelCase__ :			List[str] 						=ids_tensor([self.batch_size, self.seq_length]   ,			self.vocab_size)

         UpperCamelCase__ :			Any 						=None
         if self.use_input_mask:
            UpperCamelCase__ :			int 						=random_attention_mask([self.batch_size, self.seq_length])

         UpperCamelCase__ :			List[str] 						=None
         UpperCamelCase__ :			int 						=None
         UpperCamelCase__ :			Optional[Any] 						=None
         if self.use_labels:
            UpperCamelCase__ :			Dict 						=ids_tensor([self.batch_size]   ,			self.type_sequence_label_size)
            UpperCamelCase__ :			Dict 						=ids_tensor([self.batch_size, self.seq_length]   ,			self.num_labels)
            UpperCamelCase__ :			Optional[Any] 						=ids_tensor([self.batch_size]   ,			self.num_choices)

         UpperCamelCase__ :			Tuple 						=self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




      def    UpperCAmelCase							(		self)					->						str:







         """simple docstring"""







         UpperCamelCase__ :			Tuple 						=ids_tensor([self.batch_size, self.seq_length]   ,			self.vocab_size)

         UpperCamelCase__ :			List[str] 						=None
         if self.use_input_mask:
            UpperCamelCase__ :			Union[str, Any] 						=random_attention_mask([self.batch_size, self.seq_length])
         UpperCamelCase__ :			Union[str, Any] 						=self.get_config()

         return config, input_ids, input_mask




      def    UpperCAmelCase							(		self)					->						Optional[int]:







         """simple docstring"""







         return ErnieMConfig(
             vocab_size=self.vocab_size   ,			hidden_size=self.hidden_size   ,			num_hidden_layers=self.num_hidden_layers   ,			num_attention_heads=self.num_attention_heads   ,			intermediate_size=self.intermediate_size   ,			hidden_act=self.hidden_act   ,			hidden_dropout_prob=self.hidden_dropout_prob   ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,			max_position_embeddings=self.max_position_embeddings   ,			type_vocab_size=self.type_vocab_size   ,			initializer_range=self.initializer_range   ,			)




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						str:







         """simple docstring"""







         UpperCamelCase__ :			Tuple 						=ErnieMModel(config=__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         UpperCamelCase__ :			int 						=model(__SCREAMING_SNAKE_CASE   ,			return_dict=__SCREAMING_SNAKE_CASE)
         self.parent.assertEqual(result.last_hidden_state.shape   ,			(self.batch_size, self.seq_length, self.hidden_size))




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						List[Any]:







         """simple docstring"""







         UpperCamelCase__ :			str 						=ErnieMForQuestionAnswering(config=__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         UpperCamelCase__ :			Dict 						=model(
             __SCREAMING_SNAKE_CASE   ,			attention_mask=__SCREAMING_SNAKE_CASE   ,			start_positions=__SCREAMING_SNAKE_CASE   ,			end_positions=__SCREAMING_SNAKE_CASE   ,			)
         self.parent.assertEqual(result.start_logits.shape   ,			(self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape   ,			(self.batch_size, self.seq_length))




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						Dict:







         """simple docstring"""







         UpperCamelCase__ :			Any 						=ErnieMForInformationExtraction(config=__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         UpperCamelCase__ :			int 						=torch.ones_like(__SCREAMING_SNAKE_CASE   ,			dtype=torch.floataa)
         UpperCamelCase__ :			Union[str, Any] 						=model(
             __SCREAMING_SNAKE_CASE   ,			attention_mask=__SCREAMING_SNAKE_CASE   ,			start_positions=__SCREAMING_SNAKE_CASE   ,			end_positions=__SCREAMING_SNAKE_CASE   ,			)
         self.parent.assertEqual(result.start_logits.shape   ,			(self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape   ,			(self.batch_size, self.seq_length))




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						List[str]:







         """simple docstring"""







         UpperCamelCase__ :			List[Any] 						=self.num_labels
         UpperCamelCase__ :			List[Any] 						=ErnieMForSequenceClassification(__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         UpperCamelCase__ :			Union[str, Any] 						=model(__SCREAMING_SNAKE_CASE   ,			attention_mask=__SCREAMING_SNAKE_CASE   ,			labels=__SCREAMING_SNAKE_CASE)
         self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.num_labels))




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						Tuple:







         """simple docstring"""







         UpperCamelCase__ :			List[str] 						=self.num_labels
         UpperCamelCase__ :			int 						=ErnieMForTokenClassification(config=__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         input_ids.to(__SCREAMING_SNAKE_CASE)
         input_mask.to(__SCREAMING_SNAKE_CASE)
         token_labels.to(__SCREAMING_SNAKE_CASE)

         UpperCamelCase__ :			Tuple 						=model(__SCREAMING_SNAKE_CASE   ,			attention_mask=__SCREAMING_SNAKE_CASE   ,			labels=__SCREAMING_SNAKE_CASE)

         self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.seq_length, self.num_labels))




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						Optional[Any]:







         """simple docstring"""







         UpperCamelCase__ :			List[Any] 						=self.num_choices
         UpperCamelCase__ :			List[str] 						=ErnieMForMultipleChoice(config=__SCREAMING_SNAKE_CASE)
         model.to(__SCREAMING_SNAKE_CASE)
         model.eval()
         UpperCamelCase__ :			Optional[Any] 						=input_ids.unsqueeze(1).expand(-1   ,			self.num_choices   ,			-1).contiguous()
         UpperCamelCase__ :			int 						=input_mask.unsqueeze(1).expand(-1   ,			self.num_choices   ,			-1).contiguous()
         UpperCamelCase__ :			Optional[Any] 						=model(
             __SCREAMING_SNAKE_CASE   ,			attention_mask=__SCREAMING_SNAKE_CASE   ,			labels=__SCREAMING_SNAKE_CASE   ,			)
         self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.num_choices))




      def    UpperCAmelCase							(		self)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__ :			Tuple 						=self.prepare_config_and_inputs()
         (
             (
             UpperCamelCase__
         )						,	(
             UpperCamelCase__
         )						,	(
             UpperCamelCase__
         )						,	(
             UpperCamelCase__
         )						,	(
             UpperCamelCase__
         )						,	(
             UpperCamelCase__
         )						,	
         ) :			int 						=config_and_inputs
         UpperCamelCase__ :			Tuple 						={"input_ids": input_ids, "attention_mask": input_mask}
         return config, inputs_dict




@require_torch
class 							lowercase__(						snake_case__			,							snake_case__			,							unittest.TestCase			):






      '''simple docstring'''







      snake_case__       =					(
          (
              ErnieMModel,
              ErnieMForMultipleChoice,
              ErnieMForQuestionAnswering,
              ErnieMForSequenceClassification,
              ErnieMForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      snake_case__       =					()
      snake_case__       =					(
          {
              '''feature-extraction''': ErnieMModel,
              '''question-answering''': ErnieMForQuestionAnswering,
              '''text-classification''': ErnieMForSequenceClassification,
              '''token-classification''': ErnieMForTokenClassification,
              '''zero-shot''': ErnieMForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      snake_case__       =					False




      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE)					->						Optional[Any]:







         """simple docstring"""







         if pipeline_test_casse_name == "QAPipelineTests":
            return True

         return False




      def    UpperCAmelCase							(		self)					->						Optional[int]:







         """simple docstring"""







         UpperCamelCase__ :			Dict 						=ErnieMModelTester(self)
         UpperCamelCase__ :			int 						=ConfigTester(self   ,			config_class=__SCREAMING_SNAKE_CASE   ,			hidden_size=37)




      def    UpperCAmelCase							(		self)					->						List[Any]:







         """simple docstring"""







         self.config_tester.run_common_tests()




      def    UpperCAmelCase							(		self)					->						List[str]:







         """simple docstring"""







         UpperCamelCase__ :			Any 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__ :			int 						=self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
            UpperCamelCase__ :			List[str] 						=type
            self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						List[str]:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						Tuple:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						Any:







         """simple docstring"""







         UpperCamelCase__ :			Optional[int] 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						List[str]:







         """simple docstring"""







         UpperCamelCase__ :			Union[str, Any] 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_information_extraction(*__SCREAMING_SNAKE_CASE)




      def    UpperCAmelCase							(		self)					->						Union[str, Any]:







         """simple docstring"""







         UpperCamelCase__ :			Optional[Any] 						=self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE)




      @slow
      def    UpperCAmelCase							(		self)					->						Tuple:







         """simple docstring"""







         for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCamelCase__ :			Optional[Any] 						=ErnieMModel.from_pretrained(__SCREAMING_SNAKE_CASE)
            self.assertIsNotNone(__SCREAMING_SNAKE_CASE)




@require_torch
class 							lowercase__(						unittest.TestCase			):






      '''simple docstring'''







      @slow
      def    UpperCAmelCase							(		self)					->						Any:







         """simple docstring"""







         UpperCamelCase__ :			Optional[int] 						=ErnieMModel.from_pretrained("susnato/ernie-m-base_pytorch")
         model.eval()
         UpperCamelCase__ :			int 						=torch.tensor([[0, 1, 2, 3, 4, 5]])
         UpperCamelCase__ :			List[Any] 						=model(__SCREAMING_SNAKE_CASE)[0]

         # TODO Replace vocab size
         UpperCamelCase__ :			List[Any] 						=7_68

         UpperCamelCase__ :			Dict 						=torch.Size((1, 6, hidden_size))
         self.assertEqual(output.shape   ,			__SCREAMING_SNAKE_CASE)

         UpperCamelCase__ :			Union[str, Any] 						=torch.tensor(
             [[[-0.00_12, 0.12_45, -0.02_14], [-0.07_42, 0.02_44, -0.07_71], [-0.03_33, 0.11_64, -0.15_54]]])

         self.assertTrue(torch.allclose(output[:, :3, :3]   ,			__SCREAMING_SNAKE_CASE   ,			atol=1E-3))

