




import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
					from transformers.models.bert.modeling_flax_bert import (
					    FlaxBertForMaskedLM,
					    FlaxBertForMultipleChoice,
					    FlaxBertForNextSentencePrediction,
					    FlaxBertForPreTraining,
					    FlaxBertForQuestionAnswering,
					    FlaxBertForSequenceClassification,
					    FlaxBertForTokenClassification,
					    FlaxBertModel,
					)
class 			a__    (					unittest.TestCase      ):






						def __init__(   self		,			_A		,			_A=1_3		,			_A=7		,			_A=True		,			_A=True		,			_A=True		,			_A=True		,			_A=9_9		,			_A=3_2		,			_A=5		,			_A=4		,			_A=3_7		,			_A="gelu"		,			_A=0.1		,			_A=0.1		,			_A=5_1_2		,			_A=1_6		,			_A=2		,			_A=0.02		,			_A=4		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       parent
											__lowerCAmelCase				   =       batch_size
											__lowerCAmelCase				   =       seq_length
											__lowerCAmelCase				   =       is_training
											__lowerCAmelCase				   =       use_attention_mask
											__lowerCAmelCase				   =       use_token_type_ids
											__lowerCAmelCase				   =       use_labels
											__lowerCAmelCase				   =       vocab_size
											__lowerCAmelCase				   =       hidden_size
											__lowerCAmelCase				   =       num_hidden_layers
											__lowerCAmelCase				   =       num_attention_heads
											__lowerCAmelCase				   =       intermediate_size
											__lowerCAmelCase				   =       hidden_act
											__lowerCAmelCase				   =       hidden_dropout_prob
											__lowerCAmelCase				   =       attention_probs_dropout_prob
											__lowerCAmelCase				   =       max_position_embeddings
											__lowerCAmelCase				   =       type_vocab_size
											__lowerCAmelCase				   =       type_sequence_label_size
											__lowerCAmelCase				   =       initializer_range
											__lowerCAmelCase				   =       num_choices






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.seq_length]		,			self.vocab_size			)

											__lowerCAmelCase				   =       None
											if self.use_attention_mask:
																__lowerCAmelCase				   =       random_attention_mask([self.batch_size, self.seq_length]			)

											__lowerCAmelCase				   =       None
											if self.use_token_type_ids:
																__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.seq_length]		,			self.type_vocab_size			)

											__lowerCAmelCase				   =       BertConfig(
											    vocab_size=self.vocab_size		,			hidden_size=self.hidden_size		,			num_hidden_layers=self.num_hidden_layers		,			num_attention_heads=self.num_attention_heads		,			intermediate_size=self.intermediate_size		,			hidden_act=self.hidden_act		,			hidden_dropout_prob=self.hidden_dropout_prob		,			attention_probs_dropout_prob=self.attention_probs_dropout_prob		,			max_position_embeddings=self.max_position_embeddings		,			type_vocab_size=self.type_vocab_size		,			is_decoder=_A		,			initializer_range=self.initializer_range		,			)

											return config, input_ids, token_type_ids, attention_mask






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.prepare_config_and_inputs()
											__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase				   =       config_and_inputs
											__lowerCAmelCase				   =       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
											return config, inputs_dict






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.prepare_config_and_inputs()
											__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase				   =       config_and_inputs

											__lowerCAmelCase				   =       True
											__lowerCAmelCase				   =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
											__lowerCAmelCase				   =       ids_tensor([self.batch_size, self.seq_length]		,			vocab_size=2			)

											return (
											    config,
											    input_ids,
											    attention_mask,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)






@require_flax
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    List[str]					=    True

						_a							:    List[str]					=    (
						    (
						        FlaxBertModel,
						        FlaxBertForPreTraining,
						        FlaxBertForMaskedLM,
						        FlaxBertForMultipleChoice,
						        FlaxBertForQuestionAnswering,
						        FlaxBertForNextSentencePrediction,
						        FlaxBertForSequenceClassification,
						        FlaxBertForTokenClassification,
						        FlaxBertForQuestionAnswering,
						    )
						    if is_flax_available()
						    else ()
						)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       FlaxBertModelTester(self			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       FlaxBertModel.from_pretrained("bert-base-cased"			)
											__lowerCAmelCase				   =       model(np.ones((1, 1)			)			)
											self.assertIsNotNone(_A			)






'''simple docstring'''




import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Tuple:






         """simple docstring"""






         A       :   Union[str, Any]					=						[]
         A       :   Union[str, Any]					=						[]

         for i in range(self.num_layers	):
             A       :   Any					=						self.in_channels if i == 0 else self.out_channels

             A       :   Optional[Any]					=						FlaxResnetBlockaD(
                 in_channels=SCREAMING_SNAKE_CASE						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

             A       :   Optional[int]					=						FlaxTransformeraDModel(
                 in_channels=self.out_channels						,			n_heads=self.num_attention_heads						,			d_head=self.out_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			only_cross_attention=self.only_cross_attention						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

         A       :   Union[str, Any]					=						resnets
         A       :   Union[str, Any]					=						attentions

         if self.add_downsample:
             A       :   int					=						FlaxDownsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						()

         for resnet, attn in zip(self.resnets						,			self.attentions	):
             A       :   int					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Dict					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         if self.add_downsample:
             A       :   Optional[Any]					=						self.downsamplers_a(SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         return hidden_states, output_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						[]

         for i in range(self.num_layers	):
             A       :   Optional[Any]					=						self.in_channels if i == 0 else self.out_channels

             A       :   List[str]					=						FlaxResnetBlockaD(
                 in_channels=SCREAMING_SNAKE_CASE						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)
         A       :   Dict					=						resnets

         if self.add_downsample:
             A       :   Dict					=						FlaxDownsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Optional[Any]:






         """simple docstring"""






         A       :   str					=						()

         for resnet in self.resnets:
             A       :   Optional[int]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         if self.add_downsample:
             A       :   Optional[int]					=						self.downsamplers_a(SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         return hidden_states, output_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Tuple:






         """simple docstring"""






         A       :   Optional[Any]					=						[]
         A       :   Optional[int]					=						[]

         for i in range(self.num_layers	):
             A       :   str					=						self.in_channels if (i == self.num_layers - 1) else self.out_channels
             A       :   Dict					=						self.prev_output_channel if i == 0 else self.out_channels

             A       :   List[str]					=						FlaxResnetBlockaD(
                 in_channels=resnet_in_channels + res_skip_channels						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

             A       :   int					=						FlaxTransformeraDModel(
                 in_channels=self.out_channels						,			n_heads=self.num_attention_heads						,			d_head=self.out_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			only_cross_attention=self.only_cross_attention						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

         A       :   Dict					=						resnets
         A       :   Optional[Any]					=						attentions

         if self.add_upsample:
             A       :   Optional[int]					=						FlaxUpsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Optional[int]:






         """simple docstring"""






         for resnet, attn in zip(self.resnets						,			self.attentions	):
             # pop res hidden states
             A       :   List[str]					=						res_hidden_states_tuple[-1]
             A       :   int					=						res_hidden_states_tuple[:-1]
             A       :   List[str]					=						jnp.concatenate((hidden_states, res_hidden_states)						,			axis=-1	)

             A       :   Union[str, Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Tuple					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         if self.add_upsample:
             A       :   Dict					=						self.upsamplers_a(SCREAMING_SNAKE_CASE	)

         return hidden_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   int					=						[]

         for i in range(self.num_layers	):
             A       :   List[Any]					=						self.in_channels if (i == self.num_layers - 1) else self.out_channels
             A       :   List[str]					=						self.prev_output_channel if i == 0 else self.out_channels

             A       :   str					=						FlaxResnetBlockaD(
                 in_channels=resnet_in_channels + res_skip_channels						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

         A       :   List[Any]					=						resnets

         if self.add_upsample:
             A       :   Optional[Any]					=						FlaxUpsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Tuple:






         """simple docstring"""






         for resnet in self.resnets:
             # pop res hidden states
             A       :   Optional[int]					=						res_hidden_states_tuple[-1]
             A       :   Optional[Any]					=						res_hidden_states_tuple[:-1]
             A       :   List[Any]					=						jnp.concatenate((hidden_states, res_hidden_states)						,			axis=-1	)

             A       :   Optional[Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         if self.add_upsample:
             A       :   List[str]					=						self.upsamplers_a(SCREAMING_SNAKE_CASE	)

         return hidden_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   str					=						[
             FlaxResnetBlockaD(
                 in_channels=self.in_channels						,			out_channels=self.in_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
         ]

         A       :   List[Any]					=						[]

         for _ in range(self.num_layers	):
             A       :   int					=						FlaxTransformeraDModel(
                 in_channels=self.in_channels						,			n_heads=self.num_attention_heads						,			d_head=self.in_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

             A       :   Union[str, Any]					=						FlaxResnetBlockaD(
                 in_channels=self.in_channels						,			out_channels=self.in_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

         A       :   List[str]					=						resnets
         A       :   List[str]					=						attentions





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Dict:






         """simple docstring"""






         A       :   Optional[Any]					=						self.resnets[0](SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
         for attn, resnet in zip(self.attentions						,			self.resnets[1:]	):
             A       :   Optional[int]					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Union[str, Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         return hidden_states




