

import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MODEL_FOR_PRETRAINING_MAPPING,
			    NezhaForMaskedLM,
			    NezhaForMultipleChoice,
			    NezhaForNextSentencePrediction,
			    NezhaForPreTraining,
			    NezhaForQuestionAnswering,
			    NezhaForSequenceClassification,
			    NezhaForTokenClassification,
			    NezhaModel,
			)
			from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST






class _UpperCamelCase							:
			'''simple docstring'''







			def __init__(	self							:       Dict					,					a							:       Union[str, Any]					,					a							:       List[str]=13					,					a							:       Optional[Any]=7					,					a							:       Tuple=True					,					a							:       Dict=True					,					a							:       List[Any]=True					,					a							:       Union[str, Any]=True					,					a							:       List[str]=99					,					a							:       Any=32					,					a							:       List[str]=5					,					a							:       List[Any]=4					,					a							:       Dict=37					,					a							:       List[str]="gelu"					,					a							:       List[str]=0.1					,					a							:       Dict=0.1					,					a							:       Optional[int]=128					,					a							:       Dict=32					,					a							:       Any=16					,					a							:       Optional[Any]=2					,					a							:       Dict=0.02					,					a							:       str=3					,					a							:       Union[str, Any]=4					,					a							:       str=None					,					)							->				Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[Any]    =				parent
				SCREAMING_SNAKE_CASE      :					List[str]    =				batch_size
				SCREAMING_SNAKE_CASE      :					Any    =				seq_length
				SCREAMING_SNAKE_CASE      :					Tuple    =				is_training
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				use_input_mask
				SCREAMING_SNAKE_CASE      :					List[Any]    =				use_token_type_ids
				SCREAMING_SNAKE_CASE      :					Any    =				use_labels
				SCREAMING_SNAKE_CASE      :					Any    =				vocab_size
				SCREAMING_SNAKE_CASE      :					Tuple    =				hidden_size
				SCREAMING_SNAKE_CASE      :					Tuple    =				num_hidden_layers
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				num_attention_heads
				SCREAMING_SNAKE_CASE      :					Any    =				intermediate_size
				SCREAMING_SNAKE_CASE      :					str    =				hidden_act
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				hidden_dropout_prob
				SCREAMING_SNAKE_CASE      :					List[Any]    =				attention_probs_dropout_prob
				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				max_position_embeddings
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				type_vocab_size
				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				type_sequence_label_size
				SCREAMING_SNAKE_CASE      :					Dict    =				initializer_range
				SCREAMING_SNAKE_CASE      :					List[str]    =				num_labels
				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				num_choices
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				scope
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				ids_tensor([self.batch_size, self.seq_length]					,					self.vocab_size		)

				SCREAMING_SNAKE_CASE      :					str    =				None
				if self.use_input_mask:
					SCREAMING_SNAKE_CASE      :					Dict    =				random_attention_mask([self.batch_size, self.seq_length]		)

				SCREAMING_SNAKE_CASE      :					Tuple    =				None
				if self.use_token_type_ids:
					SCREAMING_SNAKE_CASE      :					Any    =				ids_tensor([self.batch_size, self.seq_length]					,					self.type_vocab_size		)

				SCREAMING_SNAKE_CASE      :					List[Any]    =				None
				SCREAMING_SNAKE_CASE      :					int    =				None
				SCREAMING_SNAKE_CASE      :					Dict    =				None
				if self.use_labels:
					SCREAMING_SNAKE_CASE      :					Optional[int]    =				ids_tensor([self.batch_size]					,					self.type_sequence_label_size		)
					SCREAMING_SNAKE_CASE      :					str    =				ids_tensor([self.batch_size, self.seq_length]					,					self.num_labels		)
					SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				ids_tensor([self.batch_size]					,					self.num_choices		)

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
			def 							__UpperCamelCase       (	self							:       List[Any]		)							->				str:

				"""simple docstring"""

				return NezhaConfig(
				    vocab_size=self.vocab_size					,					hidden_size=self.hidden_size					,					num_hidden_layers=self.num_hidden_layers					,					num_attention_heads=self.num_attention_heads					,					intermediate_size=self.intermediate_size					,					hidden_act=self.hidden_act					,					hidden_dropout_prob=self.hidden_dropout_prob					,					attention_probs_dropout_prob=self.attention_probs_dropout_prob					,					max_position_embeddings=self.max_position_embeddings					,					type_vocab_size=self.type_vocab_size					,					is_decoder=a					,					initializer_range=self.initializer_range					,					)
			def 							__UpperCamelCase       (	self							:       str		)							->				List[str]:

				"""simple docstring"""

				(
				    (
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,
				)      :					List[Any]    =				self.prepare_config_and_inputs()

				SCREAMING_SNAKE_CASE      :					Tuple    =				True
				SCREAMING_SNAKE_CASE      :					str    =				floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
				SCREAMING_SNAKE_CASE      :					Tuple    =				ids_tensor([self.batch_size, self.seq_length]					,					vocab_size=2		)

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    encoder_hidden_states,
				    encoder_attention_mask,
				)
			def 							__UpperCamelCase       (	self							:       Union[str, Any]					,					a							:       str					,					a							:       int					,					a							:       List[Any]					,					a							:       Tuple					,					a							:       Optional[Any]					,					a							:       Optional[int]					,					a							:       Tuple		)							->				int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				NezhaModel(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					Any    =				model(a					,					attention_mask=a					,					token_type_ids=a		)
				SCREAMING_SNAKE_CASE      :					List[Any]    =				model(a					,					token_type_ids=a		)
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				model(a		)
				self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.hidden_size)		)
				self.parent.assertEqual(result.pooler_output.shape					,					(self.batch_size, self.hidden_size)		)
			def 							__UpperCamelCase       (	self							:       Dict					,					a							:       Any					,					a							:       Tuple					,					a							:       Dict					,					a							:       str					,					a							:       List[str]					,					a							:       Optional[Any]					,					a							:       List[str]					,					a							:       Tuple					,					a							:       str					,					)							->				Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				True
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				NezhaModel(a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					int    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					encoder_hidden_states=a					,					encoder_attention_mask=a					,					)
				SCREAMING_SNAKE_CASE      :					List[Any]    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					encoder_hidden_states=a					,					)
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				model(a					,					attention_mask=a					,					token_type_ids=a		)
				self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.hidden_size)		)
				self.parent.assertEqual(result.pooler_output.shape					,					(self.batch_size, self.hidden_size)		)
			def 							__UpperCamelCase       (	self							:       Any					,					a							:       Any					,					a							:       str					,					a							:       str					,					a							:       Union[str, Any]					,					a							:       Union[str, Any]					,					a							:       str					,					a							:       Dict		)							->				Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[int]    =				NezhaForMaskedLM(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					List[Any]    =				model(a					,					attention_mask=a					,					token_type_ids=a					,					labels=a		)
				self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.vocab_size)		)
			def 							__UpperCamelCase       (	self							:       Dict					,					a							:       List[str]					,					a							:       Dict					,					a							:       Dict					,					a							:       Any					,					a							:       str					,					a							:       Optional[int]					,					a							:       Any		)							->				List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				NezhaForNextSentencePrediction(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					labels=a					,					)
				self.parent.assertEqual(result.logits.shape					,					(self.batch_size, 2)		)
			def 							__UpperCamelCase       (	self							:       Tuple					,					a							:       Tuple					,					a							:       List[Any]					,					a							:       Tuple					,					a							:       Any					,					a							:       int					,					a							:       int					,					a							:       int		)							->				Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[str]    =				NezhaForPreTraining(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					Optional[int]    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					labels=a					,					next_sentence_label=a					,					)
				self.parent.assertEqual(result.prediction_logits.shape					,					(self.batch_size, self.seq_length, self.vocab_size)		)
				self.parent.assertEqual(result.seq_relationship_logits.shape					,					(self.batch_size, 2)		)
			def 							__UpperCamelCase       (	self							:       str					,					a							:       List[Any]					,					a							:       Tuple					,					a							:       Union[str, Any]					,					a							:       Any					,					a							:       Optional[int]					,					a							:       List[Any]					,					a							:       Union[str, Any]		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				NezhaForQuestionAnswering(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					List[str]    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					start_positions=a					,					end_positions=a					,					)
				self.parent.assertEqual(result.start_logits.shape					,					(self.batch_size, self.seq_length)		)
				self.parent.assertEqual(result.end_logits.shape					,					(self.batch_size, self.seq_length)		)
			def 							__UpperCamelCase       (	self							:       Optional[int]					,					a							:       int					,					a							:       str					,					a							:       Any					,					a							:       Union[str, Any]					,					a							:       str					,					a							:       Union[str, Any]					,					a							:       Dict		)							->				List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self.num_labels
				SCREAMING_SNAKE_CASE      :					Any    =				NezhaForSequenceClassification(a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					Dict    =				model(a					,					attention_mask=a					,					token_type_ids=a					,					labels=a		)
				self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_labels)		)
			def 							__UpperCamelCase       (	self							:       Optional[Any]					,					a							:       Any					,					a							:       Tuple					,					a							:       List[Any]					,					a							:       Any					,					a							:       Dict					,					a							:       Union[str, Any]					,					a							:       int		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[str]    =				self.num_labels
				SCREAMING_SNAKE_CASE      :					Dict    =				NezhaForTokenClassification(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				model(a					,					attention_mask=a					,					token_type_ids=a					,					labels=a		)
				self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.num_labels)		)
			def 							__UpperCamelCase       (	self							:       str					,					a							:       List[str]					,					a							:       Tuple					,					a							:       Optional[int]					,					a							:       Union[str, Any]					,					a							:       str					,					a							:       Tuple					,					a							:       Union[str, Any]		)							->				str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self.num_choices
				SCREAMING_SNAKE_CASE      :					str    =				NezhaForMultipleChoice(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE      :					int    =				input_ids.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
				SCREAMING_SNAKE_CASE      :					Any    =				token_type_ids.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
				SCREAMING_SNAKE_CASE      :					Dict    =				input_mask.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				model(
				    a					,					attention_mask=a					,					token_type_ids=a					,					labels=a					,					)
				self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_choices)		)






			def 							__UpperCamelCase       (	self							:       Optional[int]		)							->				Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[Any]    =				self.prepare_config_and_inputs()
				(
				    (
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,
				)      :					Dict    =				config_and_inputs
				SCREAMING_SNAKE_CASE      :					Dict    =				{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
				return config, inputs_dict






@require_torch
class _UpperCamelCase							(							__A							,		__A							,		__A							,		unittest.TestCase      ):
			'''simple docstring'''







			lowerCamelCase__							=(
			    (
			        NezhaModel,
			        NezhaForMaskedLM,
			        NezhaForMultipleChoice,
			        NezhaForNextSentencePrediction,
			        NezhaForPreTraining,
			        NezhaForQuestionAnswering,
			        NezhaForSequenceClassification,
			        NezhaForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCamelCase__							=(
			    {
			        'feature-extraction': NezhaModel,
			        'fill-mask': NezhaForMaskedLM,
			        'question-answering': NezhaForQuestionAnswering,
			        'text-classification': NezhaForSequenceClassification,
			        'token-classification': NezhaForTokenClassification,
			        'zero-shot': NezhaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCamelCase__							=True
			def 							__UpperCamelCase       (	self							:       Optional[int]					,					a							:       Any					,					a							:       Optional[int]					,					a							:       str=False		)							->				List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					str    =				super()._prepare_for_class(a					,					a					,					return_labels=a		)

				if return_labels:
					if model_class in get_values(a		):
						SCREAMING_SNAKE_CASE      :					Any    =				torch.zeros(
						    (self.model_tester.batch_size, self.model_tester.seq_length)					,					dtype=torch.long					,					device=a		)
						SCREAMING_SNAKE_CASE      :					List[Any]    =				torch.zeros(
						    self.model_tester.batch_size					,					dtype=torch.long					,					device=a		)
				return inputs_dict
			def 							__UpperCamelCase       (	self							:       List[Any]		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Any    =				NezhaModelTester(self		)
				SCREAMING_SNAKE_CASE      :					List[Any]    =				ConfigTester(self					,					config_class=a					,					hidden_size=37		)
			def 							__UpperCamelCase       (	self							:       Union[str, Any]		)							->				Any:

				"""simple docstring"""

				self.config_tester.run_common_tests()
			def 							__UpperCamelCase       (	self							:       List[str]		)							->				Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[int]    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*a		)
			def 							__UpperCamelCase       (	self							:       Any		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_model_as_decoder(*a		)
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Optional[Any]:

				"""simple docstring"""

				(
				    (
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,(
				    SCREAMING_SNAKE_CASE
				)   ,
				)      :					Dict    =				self.model_tester.prepare_config_and_inputs_for_decoder()

				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				None

				self.model_tester.create_and_check_model_as_decoder(
				    a					,					a					,					a					,					a					,					a					,					a					,					a					,					a					,					a					,					)
			def 							__UpperCamelCase       (	self							:       int		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Tuple    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*a		)
			def 							__UpperCamelCase       (	self							:       str		)							->				int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Any    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*a		)
			def 							__UpperCamelCase       (	self							:       Tuple		)							->				int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					List[Any]    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_next_sequence_prediction(*a		)
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_pretraining(*a		)
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Tuple    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*a		)
			def 							__UpperCamelCase       (	self							:       Optional[int]		)							->				Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*a		)
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Dict    =				self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*a		)
			@slow
			def 							__UpperCamelCase       (	self							:       Union[str, Any]		)							->				Any:

				"""simple docstring"""

				for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					SCREAMING_SNAKE_CASE      :					Optional[Any]    =				NezhaModel.from_pretrained(a		)
					self.assertIsNotNone(a		)






			@slow
			@require_torch_gpu
			def 							__UpperCamelCase       (	self							:       Any		)							->				List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   ,SCREAMING_SNAKE_CASE      :					Dict    =				self.model_tester.prepare_config_and_inputs_for_common()
				for model_class in self.all_model_classes:
					# NezhaForMultipleChoice behaves incorrectly in JIT environments.
					if model_class == NezhaForMultipleChoice:
						return

					SCREAMING_SNAKE_CASE      :					List[Any]    =				True
					SCREAMING_SNAKE_CASE      :					int    =				model_class(config=a		)

					SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				self._prepare_for_class(a					,					a		)
					SCREAMING_SNAKE_CASE      :					List[Any]    =				torch.jit.trace(
					    a					,					(inputs_dict["input_ids"].to("cpu"		), inputs_dict["attention_mask"].to("cpu"		))		)

					with tempfile.TemporaryDirectory() as tmp:
						torch.jit.save(a					,					os.path.join(a					,					"bert.pt"		)		)
						SCREAMING_SNAKE_CASE      :					Tuple    =				torch.jit.load(os.path.join(a					,					"bert.pt"		)					,					map_location=a		)
						loaded(inputs_dict["input_ids"].to(a		)					,					inputs_dict["attention_mask"].to(a		)		)






@require_torch
class _UpperCamelCase							(							unittest.TestCase      ):
			'''simple docstring'''







			@slow
			def 							__UpperCamelCase       (	self							:       Dict		)							->				Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Tuple    =				NezhaModel.from_pretrained("sijunhe/nezha-cn-base"		)
				SCREAMING_SNAKE_CASE      :					Tuple    =				torch.tensor([[0, 1, 2, 3, 4, 5]]		)
				SCREAMING_SNAKE_CASE      :					Union[str, Any]    =				torch.tensor([[0, 1, 1, 1, 1, 1]]		)
				with torch.no_grad():
					SCREAMING_SNAKE_CASE      :					Any    =				model(a					,					attention_mask=a		)[0]
				SCREAMING_SNAKE_CASE      :					str    =				torch.Size((1, 6, 768)		)
				self.assertEqual(output.shape					,					a		)
				SCREAMING_SNAKE_CASE      :					Dict    =				torch.tensor([[[0.0685, 0.2441, 0.1102], [0.0600, 0.1906, 0.1349], [0.0221, 0.0819, 0.0586]]]		)

				self.assertTrue(torch.allclose(output[:, 1:4, 1:4]					,					a					,					atol=1e-4		)		)






			@slow
			def 							__UpperCamelCase       (	self							:       List[str]		)							->				Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE      :					Dict    =				NezhaForMaskedLM.from_pretrained("sijunhe/nezha-cn-base"		)
				SCREAMING_SNAKE_CASE      :					List[Any]    =				torch.tensor([[0, 1, 2, 3, 4, 5]]		)
				SCREAMING_SNAKE_CASE      :					Any    =				torch.tensor([[1, 1, 1, 1, 1, 1]]		)
				with torch.no_grad():
					SCREAMING_SNAKE_CASE      :					Optional[Any]    =				model(a					,					attention_mask=a		)[0]
				SCREAMING_SNAKE_CASE      :					Optional[Any]    =				torch.Size((1, 6, 2_1128)		)
				self.assertEqual(output.shape					,					a		)
				SCREAMING_SNAKE_CASE      :					str    =				torch.tensor(
				    [[-2.7939, -1.7902, -2.2189], [-2.8585, -1.8908, -2.3723], [-2.6499, -1.7750, -2.2558]]		)

				self.assertTrue(torch.allclose(output[:, 1:4, 1:4]					,					a					,					atol=1e-4		)		)
import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

import diffusers
from diffusers import (
    AutoencoderKL,
    EulerDiscreteScheduler,
    StableDiffusionLatentUpscalePipeline,
    StableDiffusionPipeline,
    UNetaDConditionModel,
)
from diffusers.schedulers import KarrasDiffusionSchedulers
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()




def       _snake_case	(   lowerCAmelCase							:   Tuple	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Tuple				      =    [tensor.shape for tensor in tensor_list]
		return all(shape == shapes[0] for shape in shapes[1:]	)






class 		a__						(					A__    ,		A__    ,		A__    ,		unittest.TestCase					):
					A	       =							StableDiffusionLatentUpscalePipeline
					A	       =							TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {
					    'height',
					    'width',
					    'cross_attention_kwargs',
					    'negative_prompt_embeds',
					    'prompt_embeds',
					}
					A	       =							PipelineTesterMixin.required_optional_params - {'num_images_per_prompt'}
					A	       =							TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
					A	       =							frozenset(
					    []					)  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
					A	       =							frozenset([]					)

					A	       =							True

					@property
					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Any				      =    1
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    4
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    (16, 16)

							SCREAMING_SNAKE_CASE_  :		Dict				      =    floats_tensor((batch_size, num_channels) + sizes,rng=random.Random(0    )    ).to(_A    )
							return image

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							torch.manual_seed(0    )
							SCREAMING_SNAKE_CASE_  :		str				      =    UNetaDConditionModel(
							    act_fn="gelu",attention_head_dim=8,norm_num_groups=_A,block_out_channels=[32, 32, 64, 64],time_cond_proj_dim=160,conv_in_kernel=1,conv_out_kernel=1,cross_attention_dim=32,down_block_types=(
							        "KDownBlock2D",
							        "KCrossAttnDownBlock2D",
							        "KCrossAttnDownBlock2D",
							        "KCrossAttnDownBlock2D",
							    ),in_channels=8,mid_block_type=_A,only_cross_attention=_A,out_channels=5,resnet_time_scale_shift="scale_shift",time_embedding_type="fourier",timestep_post_act="gelu",up_block_types=("KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KUpBlock2D"),)
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    AutoencoderKL(
							    block_out_channels=[32, 32, 64, 64],in_channels=3,out_channels=3,down_block_types=[
							        "DownEncoderBlock2D",
							        "DownEncoderBlock2D",
							        "DownEncoderBlock2D",
							        "DownEncoderBlock2D",
							    ],up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"],latent_channels=4,)
							SCREAMING_SNAKE_CASE_  :		int				      =    EulerDiscreteScheduler(prediction_type="sample"    )
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    CLIPTextConfig(
							    bos_token_id=0,eos_token_id=2,hidden_size=32,intermediate_size=37,layer_norm_eps=1E-05,num_attention_heads=4,num_hidden_layers=5,pad_token_id=1,vocab_size=1000,hidden_act="quick_gelu",projection_dim=512,)
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    CLIPTextModel(_A    )
							SCREAMING_SNAKE_CASE_  :		int				      =    CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"    )

							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    {
							    "unet": model.eval(),
							    "vae": vae.eval(),
							    "scheduler": scheduler,
							    "text_encoder": text_encoder,
							    "tokenizer": tokenizer,
							}

							return components

					def 				__UpperCamelCase   (    self      :       List[Any],_A      :       int,_A      :       Tuple=0    ):







							"""simple docstring"""






							if str(_A    ).startswith("mps"    ):
									SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    torch.manual_seed(_A    )
							else:
									SCREAMING_SNAKE_CASE_  :		Dict				      =    torch.Generator(device=_A    ).manual_seed(_A    )
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    {
							    "prompt": "A painting of a squirrel eating a burger",
							    "image": self.dummy_image.cpu(),
							    "generator": generator,
							    "num_inference_steps": 2,
							    "output_type": "numpy",
							}
							return inputs

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    "cpu"

							SCREAMING_SNAKE_CASE_  :		List[str]				      =    self.get_dummy_components()
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    self.pipeline_class(**_A    )
							pipe.to(_A    )
							pipe.set_progress_bar_config(disable=_A    )

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    self.get_dummy_inputs(_A    )
							SCREAMING_SNAKE_CASE_  :		Dict				      =    pipe(**_A    ).images
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    image[0, -3:, -3:, -1]

							self.assertEqual(image.shape,(1, 256, 256, 3)    )
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    np.array(
							    [0.47222412, 0.41921633, 0.44717434, 0.46874192, 0.42588258, 0.46150726, 0.4677534, 0.45583832, 0.48579055]    )
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    np.abs(image_slice.flatten() - expected_slice    ).max()
							self.assertLessEqual(_A,1E-3    )

					def 				__UpperCamelCase   (    self      :       List[Any]    ):







							"""simple docstring"""






							super().test_attention_slicing_forward_pass(expected_max_diff=7E-3    )

					def 				__UpperCamelCase   (    self      :       List[str]    ):







							"""simple docstring"""






							super().test_cpu_offload_forward_pass(expected_max_diff=3E-3    )

					def 				__UpperCamelCase   (    self      :       Tuple    ):







							"""simple docstring"""






							super().test_dict_tuple_outputs_equivalent(expected_max_difference=3E-3    )

					def 				__UpperCamelCase   (    self      :       int    ):







							"""simple docstring"""






							super().test_inference_batch_single_identical(expected_max_diff=7E-3    )

					def 				__UpperCamelCase   (    self      :       Tuple    ):







							"""simple docstring"""






							super().test_pt_np_pil_outputs_equivalent(expected_max_diff=3E-3    )

					def 				__UpperCamelCase   (    self      :       Optional[Any]    ):







							"""simple docstring"""






							super().test_save_load_local(expected_max_difference=3E-3    )

					def 				__UpperCamelCase   (    self      :       List[str]    ):







							"""simple docstring"""






							super().test_save_load_optional_components(expected_max_difference=3E-3    )






					def 				__UpperCamelCase   (    self      :       Tuple    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    [
							    "DDIMScheduler",
							    "DDPMScheduler",
							    "PNDMScheduler",
							    "HeunDiscreteScheduler",
							    "EulerAncestralDiscreteScheduler",
							    "KDPM2DiscreteScheduler",
							    "KDPM2AncestralDiscreteScheduler",
							    "DPMSolverSDEScheduler",
							]
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    self.get_dummy_components()
							SCREAMING_SNAKE_CASE_  :		Any				      =    self.pipeline_class(**_A    )

							# make sure that PNDM does not need warm-up
							pipe.scheduler.register_to_config(skip_prk_steps=_A    )

							pipe.to(_A    )
							pipe.set_progress_bar_config(disable=_A    )
							SCREAMING_SNAKE_CASE_  :		Any				      =    self.get_dummy_inputs(_A    )
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    2

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    []
							for scheduler_enum in KarrasDiffusionSchedulers:
									if scheduler_enum.name in skip_schedulers:
											# no sigma schedulers are not supported
											# no schedulers
											continue

									SCREAMING_SNAKE_CASE_  :		Tuple				      =    getattr(_A,scheduler_enum.name    )
									SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    scheduler_cls.from_config(pipe.scheduler.config    )
									SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    pipe(**_A    )[0]
									outputs.append(_A    )

							assert check_same_shape(_A    )







@require_torch_gpu
@slow
class 		a__						(					unittest.TestCase					):

					def 				__UpperCamelCase   (    self      :       Dict    ):







							"""simple docstring"""






							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()

					def 				__UpperCamelCase   (    self      :       Optional[Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    torch.manual_seed(33    )

							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4",torch_dtype=torch.floataa    )
							pipe.to("cuda"    )

							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    StableDiffusionLatentUpscalePipeline.from_pretrained(
							    "stabilityai/sd-x2-latent-upscaler",torch_dtype=torch.floataa    )
							upscaler.to("cuda"    )

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    "a photo of an astronaut high resolution, unreal engine, ultra realistic"

							SCREAMING_SNAKE_CASE_  :		str				      =    pipe(_A,generator=_A,output_type="latent"    ).images

							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    upscaler(
							    prompt=_A,image=_A,num_inference_steps=20,guidance_scale=0,generator=_A,output_type="np",).images[0]

							SCREAMING_SNAKE_CASE_  :		str				      =    load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/astronaut_1024.npy"    )
							assert np.abs((expected_image - image).mean()    ) < 5E-2






					def 				__UpperCamelCase   (    self      :       str    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    torch.manual_seed(33    )

							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    StableDiffusionLatentUpscalePipeline.from_pretrained(
							    "stabilityai/sd-x2-latent-upscaler",torch_dtype=torch.floataa    )
							upscaler.to("cuda"    )

							SCREAMING_SNAKE_CASE_  :		Any				      =    "the temple of fire by Ross Tran and Gerardo Dottori, oil on canvas"

							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    load_image(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_512.png"    )

							SCREAMING_SNAKE_CASE_  :		str				      =    upscaler(
							    prompt=_A,image=_A,num_inference_steps=20,guidance_scale=0,generator=_A,output_type="np",).images[0]

							SCREAMING_SNAKE_CASE_  :		Any				      =    load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_1024.npy"    )
							assert np.abs((expected_image - image).max()    ) < 5E-2
