





'''simple docstring'''
import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel

class      snake_case     :




 """simple docstring"""



 def __init__(       self       :							Tuple							,	__A       :							Optional[int]							,	__A       :							List[Any]=1_3							,	__A       :							Union[str, Any]=7							,	__A       :							Any=True							,	__A       :							List[Any]=True							,	__A       :							Any=True							,	__A       :							Dict=True							,	__A       :							Tuple=9_9							,	__A       :							Any=3_2							,	__A       :							Optional[int]=5							,	__A       :							Optional[int]=4							,	__A       :							List[Any]=4							,	__A       :							int="gelu"							,	__A       :							Optional[int]=0.0							,	__A       :							Dict=0.1							,	__A       :							Optional[int]=True							,	__A       :							int=5_1_2							,	__A       :							int=1_6							,	__A       :							Optional[Any]=2							,	__A       :							Optional[int]=0.02							,	__A       :							str=3							,	__A       :							List[str]=4							,	__A       :							Tuple=None							,	):
        __UpperCamelCase       =  parent
        __UpperCamelCase       =  batch_size
        __UpperCamelCase       =  seq_length
        __UpperCamelCase       =  is_training
        __UpperCamelCase       =  use_input_mask
        __UpperCamelCase       =  use_token_type_ids
        __UpperCamelCase       =  use_labels
        __UpperCamelCase       =  vocab_size
        __UpperCamelCase       =  hidden_size
        __UpperCamelCase       =  num_hidden_layers
        __UpperCamelCase       =  num_attention_heads
        __UpperCamelCase       =  intermediate_multiple_size
        __UpperCamelCase       =  hidden_act
        __UpperCamelCase       =  hidden_dropout
        __UpperCamelCase       =  attention_dropout
        __UpperCamelCase       =  weight_tying
        __UpperCamelCase       =  max_position_embeddings
        __UpperCamelCase       =  type_vocab_size
        __UpperCamelCase       =  type_sequence_label_size
        __UpperCamelCase       =  initializer_range
        __UpperCamelCase       =  num_labels
        __UpperCamelCase       =  num_choices
        __UpperCamelCase       =  scope






 def        _lowerCamelCase					(       self       :							Tuple	):
        __UpperCamelCase       =  ids_tensor([self.batch_size, self.seq_length]							,	self.vocab_size	)

        __UpperCamelCase       =  None
        if self.use_input_mask:
               __UpperCamelCase       =  random_attention_mask([self.batch_size, self.seq_length]	)

        __UpperCamelCase       =  None
        if self.use_labels:
               __UpperCamelCase       =  ids_tensor([self.batch_size, self.seq_length]							,	self.num_labels	)

        __UpperCamelCase       =  self.get_config()

        return config, input_ids, input_mask, token_labels






 def        _lowerCamelCase					(       self       :							List[str]	):
        return GPTNeoXJapaneseConfig(
            vocab_size=self.vocab_size							,	hidden_size=self.hidden_size							,	num_hidden_layers=self.num_hidden_layers							,	num_attention_heads=self.num_attention_heads							,	intermediate_multiple_size=self.intermediate_multiple_size							,	hidden_act=self.hidden_act							,	hidden_dropout=self.hidden_dropout							,	attention_dropout=self.attention_dropout							,	weight_tying=self.weight_tying							,	max_position_embeddings=self.max_position_embeddings							,	type_vocab_size=self.type_vocab_size							,	is_decoder=__A							,	initializer_range=self.initializer_range							,	)






 def        _lowerCamelCase					(       self       :							Optional[int]	):
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  self.prepare_config_and_inputs()

        __UpperCamelCase       =  True

        return config, input_ids, input_mask, token_labels






 def        _lowerCamelCase					(       self       :							Any							,	__A       :							int							,	__A       :							Union[str, Any]							,	__A       :							Union[str, Any]	):
        __UpperCamelCase       =  GPTNeoXJapaneseModel(config=__A	)
        model.to(__A	)
        model.eval()
        __UpperCamelCase       =  model(__A							,	attention_mask=__A	)
        __UpperCamelCase       =  model(__A	)
        self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.seq_length, self.hidden_size)	)






 def        _lowerCamelCase					(       self       :							List[Any]							,	__A       :							Any							,	__A       :							Any							,	__A       :							Optional[Any]	):
        __UpperCamelCase       =  True
        __UpperCamelCase       =  GPTNeoXJapaneseModel(__A	)
        model.to(__A	)
        model.eval()
        __UpperCamelCase       =  model(__A							,	attention_mask=__A	)
        self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.seq_length, self.hidden_size)	)






 def        _lowerCamelCase					(       self       :							Optional[int]							,	__A       :							List[Any]							,	__A       :							Union[str, Any]							,	__A       :							Tuple							,	__A       :							Optional[Any]	):
        __UpperCamelCase       =  GPTNeoXJapaneseForCausalLM(config=__A	)
        model.to(__A	)
        model.eval()
        __UpperCamelCase       =  model(__A							,	attention_mask=__A							,	labels=__A	)
        self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.seq_length, self.vocab_size)	)






 def        _lowerCamelCase					(       self       :							Optional[Any]							,	__A       :							List[Any]							,	__A       :							Union[str, Any]							,	__A       :							str	):
        __UpperCamelCase       =  True
        __UpperCamelCase       =  GPTNeoXJapaneseForCausalLM(config=__A	)
        model.to(__A	)
        model.eval()

        # first forward pass
        __UpperCamelCase       =  model(__A							,	attention_mask=__A							,	use_cache=__A	)
        __UpperCamelCase       =  outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        __UpperCamelCase       =  ids_tensor((self.batch_size, 3)							,	config.vocab_size	)
        __UpperCamelCase       =  ids_tensor((self.batch_size, 3)							,	vocab_size=2	)

        # append to next input_ids and
        __UpperCamelCase       =  torch.cat([input_ids, next_tokens]							,	dim=-1	)
        __UpperCamelCase       =  torch.cat([input_mask, next_mask]							,	dim=-1	)

        __UpperCamelCase       =  model(__A							,	attention_mask=__A							,	output_hidden_states=__A	)
        __UpperCamelCase       =  output_from_no_past['hidden_states'][0]
        __UpperCamelCase       =  model(
            __A							,	attention_mask=__A							,	past_key_values=__A							,	output_hidden_states=__A							,	)['hidden_states'][0]

        # select random slice
        __UpperCamelCase       =  ids_tensor((1,)							,	output_from_past.shape[-1]	).item()
        __UpperCamelCase       =  output_from_no_past[:, -3:, random_slice_idx].detach()
        __UpperCamelCase       =  output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(__A							,	__A							,	atol=1e-3	)	)






 def        _lowerCamelCase					(       self       :							Union[str, Any]	):
        __UpperCamelCase       =  self.prepare_config_and_inputs()
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  config_and_inputs
        __UpperCamelCase       =  {'input_ids': input_ids, 'attention_mask': input_mask}
        return config, inputs_dict






@require_torch
class      snake_case     (    __lowerCamelCase			,       __lowerCamelCase			,       unittest.TestCase       ):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_	:					Union[str, Any]					=(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
 SCREAMING_SNAKE_CASE_	:					List[Any]					=(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
 SCREAMING_SNAKE_CASE_	:					Optional[int]					=(
     {"feature-extraction": GPTNeoXJapaneseModel, "text-generation": GPTNeoXJapaneseForCausalLM}
     if is_torch_available()
     else {}
 )
 SCREAMING_SNAKE_CASE_	:					Optional[int]					=False
 SCREAMING_SNAKE_CASE_	:					Any					=False
 SCREAMING_SNAKE_CASE_	:					Optional[int]					=False
 SCREAMING_SNAKE_CASE_	:					int					=False






 def        _lowerCamelCase					(       self       :							Dict	):
        __UpperCamelCase       =  GPTNeoXJapaneseModelTester(self	)
        __UpperCamelCase       =  ConfigTester(self							,	config_class=__A							,	hidden_size=3_7	)






 def        _lowerCamelCase					(       self       :							str	):
        self.config_tester.run_common_tests()






 def        _lowerCamelCase					(       self       :							Optional[Any]	):
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(__A							,	__A							,	__A	)






 def        _lowerCamelCase					(       self       :							Union[str, Any]	):
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(__A							,	__A							,	__A	)






 def        _lowerCamelCase					(       self       :							int	):
        # This regression test was failing with PyTorch < 1.3
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  self.model_tester.prepare_config_and_inputs_for_decoder()

        __UpperCamelCase       =  None

        self.model_tester.create_and_check_model_as_decoder(__A							,	__A							,	__A	)






 def        _lowerCamelCase					(       self       :							Optional[int]	):
        __UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       ,						__UpperCamelCase       =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(__A							,	__A							,	__A	)






 def        _lowerCamelCase					(       self       :							Dict	):
        __UpperCamelCase       =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_causal_lm(*__A	)






 @slow
 def        _lowerCamelCase					(       self       :							Optional[int]	):
        __UpperCamelCase       =  'abeja/gpt-neox-japanese-2.7b'

        __UpperCamelCase       =  ['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

        __UpperCamelCase       =  [
            'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
            '100年後に必要とされる会社は、「人」が中心の会社です。',
            'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
            '国境の長いトンネルを抜けると、そこは雪国だった。',
            '美味しい日本食といえば、やっぱりお寿司ですよね。',
        ]

        __UpperCamelCase       =  GPTNeoXJapaneseTokenizer.from_pretrained(__A	)
        __UpperCamelCase       =  GPTNeoXJapaneseForCausalLM.from_pretrained(__A	)

        __UpperCamelCase       =  []
        for prompt in prompts:
               __UpperCamelCase       =  tokenizer(__A							,	return_tensors='pt'	).input_ids
               __UpperCamelCase       =  model.generate(__A							,	max_length=5_0	)
               __UpperCamelCase       =  tokenizer.batch_decode(__A							,	skip_special_tokens=__A	)
               predicted_outputs += generated_string
        self.assertListEqual(__A							,	__A	)







'''simple docstring'''
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


a__							:							Any =logging.get_logger(__name__)

a__							:							Optional[Any] ={
    '''EleutherAI/gpt-neo-1.3B''': '''https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json''',
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}

class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_	:					Dict					="gpt_neo"
 SCREAMING_SNAKE_CASE_	:					Optional[int]					=["past_key_values"]
 SCREAMING_SNAKE_CASE_	:					List[Any]					={"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}






 def __init__(       self       :							Union[str, Any]							,	__A       :							Union[str, Any]=5_0_2_5_7							,	__A       :							Any=2_0_4_8							,	__A       :							Optional[Any]=2_0_4_8							,	__A       :							Any=2_4							,	__A       :							Union[str, Any]=[[["global", "local"], 1_2]]							,	__A       :							str=1_6							,	__A       :							Optional[int]=None							,	__A       :							Union[str, Any]=2_5_6							,	__A       :							Any="gelu_new"							,	__A       :							Dict=0.0							,	__A       :							Optional[int]=0.0							,	__A       :							int=0.0							,	__A       :							List[str]=0.1							,	__A       :							Any=1e-5							,	__A       :							int=0.02							,	__A       :							List[str]=True							,	__A       :							Tuple=5_0_2_5_6							,	__A       :							Optional[Any]=5_0_2_5_6							,	**__A       :							Optional[Any]							,	):
        __UpperCamelCase       =  vocab_size
        __UpperCamelCase       =  max_position_embeddings
        __UpperCamelCase       =  hidden_size
        __UpperCamelCase       =  num_layers
        __UpperCamelCase       =  num_heads
        __UpperCamelCase       =  intermediate_size
        __UpperCamelCase       =  window_size
        __UpperCamelCase       =  activation_function
        __UpperCamelCase       =  resid_dropout
        __UpperCamelCase       =  embed_dropout
        __UpperCamelCase       =  attention_dropout
        __UpperCamelCase       =  classifier_dropout
        __UpperCamelCase       =  layer_norm_epsilon
        __UpperCamelCase       =  initializer_range
        __UpperCamelCase       =  use_cache

        __UpperCamelCase       =  bos_token_id
        __UpperCamelCase       =  eos_token_id

        __UpperCamelCase       =  attention_types
        __UpperCamelCase       =  self.expand_attention_types_params(__A	)

        if len(self.attention_layers	) != self.num_layers:
               raise ValueError(
                   'Configuration for convolutional module is incorrect. '
                   'It is required that `len(config.attention_layers)` == `config.num_layers` '
                   f'''but is `len(config.attention_layers) = {len(self.attention_layers	)}`, '''
                   f'''`config.num_layers = {self.num_layers}`. '''
                   '`config.attention_layers` is prepared using `config.attention_types`. '
                   'Please verify the value of `config.attention_types` argument.'	)

        super().__init__(bos_token_id=__A							,	eos_token_id=__A							,	**__A	)






 @staticmethod
 def        _lowerCamelCase					(       __A       :							Tuple	):
        __UpperCamelCase       =  []
        for item in attention_types:
               for _ in range(item[1]	):
                      attentions.extend(item[0]	)
        return attentions






def 				lowercase__   (   __lowercase						:  Tuple				,  __lowercase						:  Any				,  __lowercase						:  Union[str, Any]				,  __lowercase						:  List[str]     ) ->      Any:



       """simple docstring"""






       import torch

       __UpperCamelCase       =  input.size()
       __UpperCamelCase       =  len(__lowercase     )
       __UpperCamelCase       =  shape[dimension]

       __UpperCamelCase       =  torch.arange(0				,  __lowercase				,  __lowercase     )
       __UpperCamelCase       =  torch.div(sizedim - size				,  __lowercase				,  rounding_mode='floor'     ) + 1
       __UpperCamelCase       =  torch.arange(__lowercase     ) + low_indices[:min_length][:, None]

       __UpperCamelCase       =  [slice(__lowercase     )] * rank
       __UpperCamelCase       =  indices
       __UpperCamelCase       =  input[s]

       __UpperCamelCase       =  list(range(0				,  rank + 1     )     )
       perm.append(perm.pop(dimension + 1     )     )

       return sliced.permute(__lowercase     )
def 				lowercase__   (   __lowercase						:  Union[str, Any]				,  __lowercase						:  Optional[int]     ) ->      Optional[int]:



       """simple docstring"""






       import torch

       __UpperCamelCase       =  torch.arange(1				,  __lowercase     )
       __UpperCamelCase       =  torch.remainder(__lowercase				,  __lowercase     )
       __UpperCamelCase       =  remainders == 0
       __UpperCamelCase       =  candidates[divisor_indices]
       __UpperCamelCase       =  torch.max(__lowercase     )
       return largest_divisor, torch.div(__lowercase				,  __lowercase				,  rounding_mode='floor'     )


class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 @property
 def        _lowerCamelCase					(       self       :							Tuple	):
        __UpperCamelCase       =  OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}	)
        if self.use_past:
               self.fill_with_past_key_values_(__A							,	direction='inputs'	)
               __UpperCamelCase       =  {0: 'batch', 1: 'past_sequence + sequence'}
        else:
               __UpperCamelCase       =  {0: 'batch', 1: 'sequence'}

        return common_inputs






 @property
 def        _lowerCamelCase					(       self       :							int	):
        return self._config.num_heads






 def        _lowerCamelCase					(       self       :							List[str]							,	__A       :							PreTrainedTokenizer							,	__A       :							int = -1							,	__A       :							int = -1							,	__A       :							bool = False							,	__A       :							Optional[TensorType] = None							,	):
        __UpperCamelCase       =  super(__A							,	self	).generate_dummy_inputs(
            __A							,	batch_size=__A							,	seq_length=__A							,	is_pair=__A							,	framework=__A	)

        # We need to order the input in the way they appears in the forward()
        __UpperCamelCase       =  OrderedDict({'input_ids': common_inputs['input_ids']}	)

        # Need to add the past_keys
        if self.use_past:
               if not is_torch_available():
                      raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'	)
               else:
                      import torch

                      __UpperCamelCase       ,						__UpperCamelCase       =  common_inputs['input_ids'].shape
                      # Not using the same length for past_key_values
                      __UpperCamelCase       =  seqlen + 2
                      __UpperCamelCase       =  (
                          batch,
                          self.num_attention_heads,
                          past_key_values_length,
                          self._config.hidden_size // self.num_attention_heads,
                      )
                      __UpperCamelCase       =  [
                          (torch.zeros(__A	), torch.zeros(__A	)) for _ in range(self.num_layers	)
                      ]

        __UpperCamelCase       =  common_inputs['attention_mask']
        if self.use_past:
               __UpperCamelCase       =  ordered_inputs['attention_mask'].dtype
               __UpperCamelCase       =  torch.cat(
                   [ordered_inputs['attention_mask'], torch.ones(__A							,	__A							,	dtype=__A	)]							,	dim=1	)

        return ordered_inputs






 @property
 def        _lowerCamelCase					(       self       :							Dict	):
        return 1_3
