




'''simple docstring'''



import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel






class 				_a							:





       '''simple docstring'''






       def __init__(	self,				A,				A=13,				A=7,				A=True,				A=True,				A=True,				A=True,				A=99,				A=32,				A=5,				A=4,				A=4,				A="gelu",				A=0.0,				A=0.1,				A=True,				A=512,				A=16,				A=2,				A=0.02,				A=3,				A=4,				A=None,				):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		parent
             SCREAMING_SNAKE_CASE					:	List[str]   =		batch_size
             SCREAMING_SNAKE_CASE					:	str   =		seq_length
             SCREAMING_SNAKE_CASE					:	int   =		is_training
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		use_input_mask
             SCREAMING_SNAKE_CASE					:	Optional[int]   =		use_token_type_ids
             SCREAMING_SNAKE_CASE					:	Dict   =		use_labels
             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		vocab_size
             SCREAMING_SNAKE_CASE					:	int   =		hidden_size
             SCREAMING_SNAKE_CASE					:	Tuple   =		num_hidden_layers
             SCREAMING_SNAKE_CASE					:	int   =		num_attention_heads
             SCREAMING_SNAKE_CASE					:	Dict   =		intermediate_multiple_size
             SCREAMING_SNAKE_CASE					:	Optional[int]   =		hidden_act
             SCREAMING_SNAKE_CASE					:	str   =		hidden_dropout
             SCREAMING_SNAKE_CASE					:	int   =		attention_dropout
             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		weight_tying
             SCREAMING_SNAKE_CASE					:	Any   =		max_position_embeddings
             SCREAMING_SNAKE_CASE					:	Any   =		type_vocab_size
             SCREAMING_SNAKE_CASE					:	List[Any]   =		type_sequence_label_size
             SCREAMING_SNAKE_CASE					:	int   =		initializer_range
             SCREAMING_SNAKE_CASE					:	Tuple   =		num_labels
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		num_choices
             SCREAMING_SNAKE_CASE					:	List[str]   =		scope







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Optional[int]   =		ids_tensor([self.batch_size, self.seq_length],				self.vocab_size   )

             SCREAMING_SNAKE_CASE					:	Any   =		None
             if self.use_input_mask:
                   SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		random_attention_mask([self.batch_size, self.seq_length]   )

             SCREAMING_SNAKE_CASE					:	Optional[int]   =		None
             if self.use_labels:
                   SCREAMING_SNAKE_CASE					:	Dict   =		ids_tensor([self.batch_size, self.seq_length],				self.num_labels   )

             SCREAMING_SNAKE_CASE					:	Any   =		self.get_config()

             return config, input_ids, input_mask, token_labels







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             return GPTNeoXJapaneseConfig(
                 vocab_size=self.vocab_size,				hidden_size=self.hidden_size,				num_hidden_layers=self.num_hidden_layers,				num_attention_heads=self.num_attention_heads,				intermediate_multiple_size=self.intermediate_multiple_size,				hidden_act=self.hidden_act,				hidden_dropout=self.hidden_dropout,				attention_dropout=self.attention_dropout,				weight_tying=self.weight_tying,				max_position_embeddings=self.max_position_embeddings,				type_vocab_size=self.type_vocab_size,				is_decoder=UpperCamelCase_,				initializer_range=self.initializer_range,				)







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	List[Any]   =		self.prepare_config_and_inputs()

             SCREAMING_SNAKE_CASE					:	List[Any]   =		True

             return config, input_ids, input_mask, token_labels







       def     UpperCamelCase_				(	self,				A,				A,				A   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Optional[int]   =		GPTNeoXJapaneseModel(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             SCREAMING_SNAKE_CASE					:	Tuple   =		model(UpperCamelCase_,				attention_mask=UpperCamelCase_   )
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		model(UpperCamelCase_   )
             self.parent.assertEqual(result.last_hidden_state.shape,				(self.batch_size, self.seq_length, self.hidden_size)   )







       def     UpperCamelCase_				(	self,				A,				A,				A   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Dict   =		True
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		GPTNeoXJapaneseModel(UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             SCREAMING_SNAKE_CASE					:	Any   =		model(UpperCamelCase_,				attention_mask=UpperCamelCase_   )
             self.parent.assertEqual(result.last_hidden_state.shape,				(self.batch_size, self.seq_length, self.hidden_size)   )







       def     UpperCamelCase_				(	self,				A,				A,				A,				A   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	List[Any]   =		GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             SCREAMING_SNAKE_CASE					:	Any   =		model(UpperCamelCase_,				attention_mask=UpperCamelCase_,				labels=UpperCamelCase_   )
             self.parent.assertEqual(result.logits.shape,				(self.batch_size, self.seq_length, self.vocab_size)   )







       def     UpperCamelCase_				(	self,				A,				A,				A   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Any   =		True
             SCREAMING_SNAKE_CASE					:	Tuple   =		GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()

             # first forward pass
             SCREAMING_SNAKE_CASE					:	str   =		model(UpperCamelCase_,				attention_mask=UpperCamelCase_,				use_cache=UpperCamelCase_   )
             SCREAMING_SNAKE_CASE					:	int   =		outputs.past_key_values

             # create hypothetical multiple next token and extent to next_input_ids
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		ids_tensor((self.batch_size, 3),				config.vocab_size   )
             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		ids_tensor((self.batch_size, 3),				vocab_size=2   )

             # append to next input_ids and
             SCREAMING_SNAKE_CASE					:	Any   =		torch.cat([input_ids, next_tokens],				dim=-1   )
             SCREAMING_SNAKE_CASE					:	int   =		torch.cat([input_mask, next_mask],				dim=-1   )

             SCREAMING_SNAKE_CASE					:	List[Any]   =		model(UpperCamelCase_,				attention_mask=UpperCamelCase_,				output_hidden_states=UpperCamelCase_   )
             SCREAMING_SNAKE_CASE					:	List[str]   =		output_from_no_past['''hidden_states'''][0]
             SCREAMING_SNAKE_CASE					:	Tuple   =		model(
                 UpperCamelCase_,				attention_mask=UpperCamelCase_,				past_key_values=UpperCamelCase_,				output_hidden_states=UpperCamelCase_,				)['''hidden_states'''][0]

             # select random slice
             SCREAMING_SNAKE_CASE					:	Any   =		ids_tensor((1,),				output_from_past.shape[-1]   ).item()
             SCREAMING_SNAKE_CASE					:	List[Any]   =		output_from_no_past[:, -3:, random_slice_idx].detach()
             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		output_from_past[:, :, random_slice_idx].detach()

             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(UpperCamelCase_,				UpperCamelCase_,				atol=1E-3   )   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	List[Any]   =		self.prepare_config_and_inputs()
             SCREAMING_SNAKE_CASE					:	List[Any]   =		config_and_inputs
             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
             return config, inputs_dict






@require_torch
class 				_a							(       a__      ,	a__      ,	unittest.TestCase      ):





       '''simple docstring'''






       A		:							Optional[Any]								=		(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
       A		:							List[Any]								=		(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
       A		:							Tuple								=		(
           {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
           if is_torch_available()
           else {}
       )
       A		:							Tuple								=		False
       A		:							Optional[int]								=		False
       A		:							Optional[Any]								=		False
       A		:							Tuple								=		False







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		GPTNeoXJapaneseModelTester(self   )
             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		ConfigTester(self,				config_class=UpperCamelCase_,				hidden_size=37   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             self.config_tester.run_common_tests()







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	str   =		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(UpperCamelCase_,				UpperCamelCase_,				UpperCamelCase_   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	List[str]   =		self.model_tester.prepare_config_and_inputs_for_decoder()
             self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_,				UpperCamelCase_,				UpperCamelCase_   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	int   =		self.model_tester.prepare_config_and_inputs_for_decoder()

             SCREAMING_SNAKE_CASE					:	Dict   =		None

             self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_,				UpperCamelCase_,				UpperCamelCase_   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_decoder_model_past_large_inputs(UpperCamelCase_,				UpperCamelCase_,				UpperCamelCase_   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_causal_lm(*UpperCamelCase_   )







       @slow
       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Optional[int]   =		'''abeja/gpt-neox-japanese-2.7b'''

             SCREAMING_SNAKE_CASE					:	int   =		['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

             SCREAMING_SNAKE_CASE					:	List[Any]   =		[
                 '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
                 '''100年後に必要とされる会社は、「人」が中心の会社です。''',
                 '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
                 '''国境の長いトンネルを抜けると、そこは雪国だった。''',
                 '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
             ]

             SCREAMING_SNAKE_CASE					:	int   =		GPTNeoXJapaneseTokenizer.from_pretrained(UpperCamelCase_   )
             SCREAMING_SNAKE_CASE					:	Tuple   =		GPTNeoXJapaneseForCausalLM.from_pretrained(UpperCamelCase_   )

             SCREAMING_SNAKE_CASE					:	List[Any]   =		[]
             for prompt in prompts:
                   SCREAMING_SNAKE_CASE					:	str   =		tokenizer(UpperCamelCase_,				return_tensors='pt'   ).input_ids
                   SCREAMING_SNAKE_CASE					:	Dict   =		model.generate(UpperCamelCase_,				max_length=50   )
                   SCREAMING_SNAKE_CASE					:	Any   =		tokenizer.batch_decode(UpperCamelCase_,				skip_special_tokens=UpperCamelCase_   )
                   predicted_outputs += generated_string
             self.assertListEqual(UpperCamelCase_,				UpperCamelCase_   )






"""simple docstring"""





import unittest

import numpy as np

from transformers import RobertaPreLayerNormConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
 import jax.numpy as jnp

 from transformers.models.roberta_prelayernorm.modeling_flax_roberta_prelayernorm import (
     FlaxRobertaPreLayerNormForCausalLM,
     FlaxRobertaPreLayerNormForMaskedLM,
     FlaxRobertaPreLayerNormForMultipleChoice,
     FlaxRobertaPreLayerNormForQuestionAnswering,
     FlaxRobertaPreLayerNormForSequenceClassification,
     FlaxRobertaPreLayerNormForTokenClassification,
     FlaxRobertaPreLayerNormModel,
 )



class 						snake_case_(				unittest.TestCase	):






     def __init__(		self     :    List[Any]    , UpperCamelCase_     :    Union[str, Any]    , UpperCamelCase_     :    List[Any]=1_3    , UpperCamelCase_     :    Tuple=7    , UpperCamelCase_     :    List[Any]=True    , UpperCamelCase_     :    int=True    , UpperCamelCase_     :    Union[str, Any]=True    , UpperCamelCase_     :    Optional[Any]=True    , UpperCamelCase_     :    List[str]=9_9    , UpperCamelCase_     :    str=3_2    , UpperCamelCase_     :    Union[str, Any]=5    , UpperCamelCase_     :    int=4    , UpperCamelCase_     :    Optional[Any]=3_7    , UpperCamelCase_     :    Optional[int]="gelu"    , UpperCamelCase_     :    Any=0.1    , UpperCamelCase_     :    List[str]=0.1    , UpperCamelCase_     :    str=5_1_2    , UpperCamelCase_     :    Optional[Any]=1_6    , UpperCamelCase_     :    Union[str, Any]=2    , UpperCamelCase_     :    Any=0.02    , UpperCamelCase_     :    Union[str, Any]=4    , ):
           lowerCAmelCase    :		str				=      parent
           lowerCAmelCase    :		List[str]				=      batch_size
           lowerCAmelCase    :		int				=      seq_length
           lowerCAmelCase    :		str				=      is_training
           lowerCAmelCase    :		Tuple				=      use_attention_mask
           lowerCAmelCase    :		Dict				=      use_token_type_ids
           lowerCAmelCase    :		Optional[int]				=      use_labels
           lowerCAmelCase    :		Optional[Any]				=      vocab_size
           lowerCAmelCase    :		Optional[int]				=      hidden_size
           lowerCAmelCase    :		Optional[Any]				=      num_hidden_layers
           lowerCAmelCase    :		str				=      num_attention_heads
           lowerCAmelCase    :		Optional[Any]				=      intermediate_size
           lowerCAmelCase    :		int				=      hidden_act
           lowerCAmelCase    :		int				=      hidden_dropout_prob
           lowerCAmelCase    :		Tuple				=      attention_probs_dropout_prob
           lowerCAmelCase    :		str				=      max_position_embeddings
           lowerCAmelCase    :		str				=      type_vocab_size
           lowerCAmelCase    :		str				=      type_sequence_label_size
           lowerCAmelCase    :		Any				=      initializer_range
           lowerCAmelCase    :		int				=      num_choices






     def    lowerCamelCase__   (		self     :    Optional[int] ):
           lowerCAmelCase    :		Tuple				=      ids_tensor([self.batch_size, self.seq_length]    , self.vocab_size )

           lowerCAmelCase    :		Optional[int]				=      None
           if self.use_attention_mask:
                 lowerCAmelCase    :		Union[str, Any]				=      random_attention_mask([self.batch_size, self.seq_length] )

           lowerCAmelCase    :		Union[str, Any]				=      None
           if self.use_token_type_ids:
                 lowerCAmelCase    :		Dict				=      ids_tensor([self.batch_size, self.seq_length]    , self.type_vocab_size )

           lowerCAmelCase    :		Union[str, Any]				=      RobertaPreLayerNormConfig(
               vocab_size=self.vocab_size    , hidden_size=self.hidden_size    , num_hidden_layers=self.num_hidden_layers    , num_attention_heads=self.num_attention_heads    , intermediate_size=self.intermediate_size    , hidden_act=self.hidden_act    , hidden_dropout_prob=self.hidden_dropout_prob    , attention_probs_dropout_prob=self.attention_probs_dropout_prob    , max_position_embeddings=self.max_position_embeddings    , type_vocab_size=self.type_vocab_size    , is_decoder=UpperCamelCase_    , initializer_range=self.initializer_range    , )

           return config, input_ids, token_type_ids, attention_mask






     def    lowerCamelCase__   (		self     :    int ):
           lowerCAmelCase    :		List[str]				=      self.prepare_config_and_inputs()
           lowerCAmelCase,					lowerCAmelCase,					lowerCAmelCase,					lowerCAmelCase    :		Optional[Any]				=      config_and_inputs
           lowerCAmelCase    :		Optional[Any]				=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
           return config, inputs_dict






     def    lowerCamelCase__   (		self     :    List[str] ):
           lowerCAmelCase    :		int				=      self.prepare_config_and_inputs()
           lowerCAmelCase,					lowerCAmelCase,					lowerCAmelCase,					lowerCAmelCase    :		Tuple				=      config_and_inputs

           lowerCAmelCase    :		str				=      True
           lowerCAmelCase    :		Optional[Any]				=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size] )
           lowerCAmelCase    :		str				=      ids_tensor([self.batch_size, self.seq_length]    , vocab_size=2 )

           return (
               config,
               input_ids,
               token_type_ids,
               encoder_hidden_states,
               encoder_attention_mask,
           )



@require_flax
# Copied from tests.models.roberta.test_modelling_flax_roberta.FlaxRobertaPreLayerNormModelTest with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta-base->andreasmadsen/efficient_mlm_m0.40
class 						snake_case_(				a__							,    unittest.TestCase	):
     __UpperCamelCase     =    True

     __UpperCamelCase     =    (
         (
             FlaxRobertaPreLayerNormModel,
             FlaxRobertaPreLayerNormForCausalLM,
             FlaxRobertaPreLayerNormForMaskedLM,
             FlaxRobertaPreLayerNormForSequenceClassification,
             FlaxRobertaPreLayerNormForTokenClassification,
             FlaxRobertaPreLayerNormForMultipleChoice,
             FlaxRobertaPreLayerNormForQuestionAnswering,
         )
         if is_flax_available()
         else ()
     )






     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		Any				=      FlaxRobertaPreLayerNormModelTester(self )






     @slow
     def    lowerCamelCase__   (		self     :    List[str] ):
           for model_class_name in self.all_model_classes:
                 lowerCAmelCase    :		Optional[int]				=      model_class_name.from_pretrained('''andreasmadsen/efficient_mlm_m0.40'''    , from_pt=UpperCamelCase_ )
                 lowerCAmelCase    :		int				=      model(np.ones((1, 1) ) )
                 self.assertIsNotNone(UpperCamelCase_ )



@require_flax
class 						snake_case_(				unittest.TestCase	):






     @slow
     def    lowerCamelCase__   (		self     :    List[str] ):
           lowerCAmelCase    :		str				=      FlaxRobertaPreLayerNormForMaskedLM.from_pretrained('''andreasmadsen/efficient_mlm_m0.40'''    , from_pt=UpperCamelCase_ )

           lowerCAmelCase    :		Any				=      np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]    , dtype=jnp.intaa )
           lowerCAmelCase    :		Union[str, Any]				=      model(UpperCamelCase_ )[0]
           lowerCAmelCase    :		str				=      [1, 1_1, 5_0_2_6_5]
           self.assertEqual(list(output.shape )    , UpperCamelCase_ )
           # compare the actual values for a slice.
           lowerCAmelCase    :		Optional[Any]				=      np.array(
               [[[40.4_880, 18.0_199, -5.2_367], [-1.8_877, -4.0_885, 10.7_085], [-2.2_613, -5.6_110, 7.2_665]]]    , dtype=np.floataa )
           self.assertTrue(np.allclose(output[:, :3, :3]    , UpperCamelCase_    , atol=1E-4 ) )






     @slow
     def    lowerCamelCase__   (		self     :    List[str] ):
           lowerCAmelCase    :		Dict				=      FlaxRobertaPreLayerNormModel.from_pretrained('''andreasmadsen/efficient_mlm_m0.40'''    , from_pt=UpperCamelCase_ )

           lowerCAmelCase    :		str				=      np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]    , dtype=jnp.intaa )
           lowerCAmelCase    :		str				=      model(UpperCamelCase_ )[0]
           # compare the actual values for a slice.
           lowerCAmelCase    :		str				=      np.array(
               [[[0.0_208, -0.0_356, 0.0_237], [-0.1_569, -0.0_411, -0.2_626], [0.1_879, 0.0_125, -0.0_089]]]    , dtype=np.floataa )
           self.assertTrue(np.allclose(output[:, :3, :3]    , UpperCamelCase_    , atol=1E-4 ) )


