

import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch
     from torch import nn

     from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
     from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
     from PIL import Image

     from transformers import AutoImageProcessor


class  A						:







    """simple docstring"""





    def __init__(      self :       Any,lowercase_ :       List[str],lowercase_ :       Any=1_3,lowercase_ :       Union[str, Any]=3_2,lowercase_ :       Dict=2,lowercase_ :       str=3,lowercase_ :       List[str]=1_6,lowercase_ :       Optional[Any]=[1, 2, 1],lowercase_ :       List[str]=[2, 2, 4],lowercase_ :       Optional[int]=2,lowercase_ :       str=2.0,lowercase_ :       Optional[int]=True,lowercase_ :       Any=0.0,lowercase_ :       Optional[int]=0.0,lowercase_ :       Union[str, Any]=0.1,lowercase_ :       int="gelu",lowercase_ :       List[Any]=False,lowercase_ :       Any=True,lowercase_ :       Any=0.02,lowercase_ :       str=1E-5,lowercase_ :       Tuple=True,lowercase_ :       Dict=None,lowercase_ :       int=True,lowercase_ :       Union[str, Any]=1_0,lowercase_ :       Tuple=8,)->    List[Any]:





        '''simple docstring'''







        A__          =   parent
        A__          =   batch_size
        A__          =   image_size
        A__          =   patch_size
        A__          =   num_channels
        A__          =   embed_dim
        A__          =   depths
        A__          =   num_heads
        A__          =   window_size
        A__          =   mlp_ratio
        A__          =   qkv_bias
        A__          =   hidden_dropout_prob
        A__          =   attention_probs_dropout_prob
        A__          =   drop_path_rate
        A__          =   hidden_act
        A__          =   use_absolute_embeddings
        A__          =   patch_norm
        A__          =   layer_norm_eps
        A__          =   initializer_range
        A__          =   is_training
        A__          =   scope
        A__          =   use_labels
        A__          =   type_sequence_label_size
        A__          =   encoder_stride


    def 						snake_case__     (      self :       Dict		)->    Any:





        '''simple docstring'''







        A__          =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

        A__          =   None
        if self.use_labels:
            A__          =   ids_tensor([self.batch_size],self.type_sequence_label_size		)

        A__          =   self.get_config()

        return config, pixel_values, labels


    def 						snake_case__     (      self :       Union[str, Any]		)->    Tuple:





        '''simple docstring'''







        return SwinvaConfig(
            image_size=self.image_size,patch_size=self.patch_size,num_channels=self.num_channels,embed_dim=self.embed_dim,depths=self.depths,num_heads=self.num_heads,window_size=self.window_size,mlp_ratio=self.mlp_ratio,qkv_bias=self.qkv_bias,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,drop_path_rate=self.drop_path_rate,hidden_act=self.hidden_act,use_absolute_embeddings=self.use_absolute_embeddings,path_norm=self.patch_norm,layer_norm_eps=self.layer_norm_eps,initializer_range=self.initializer_range,encoder_stride=self.encoder_stride,)


    def 						snake_case__     (      self :       str,lowercase_ :       str,lowercase_ :       Any,lowercase_ :       int		)->    Dict:





        '''simple docstring'''







        A__          =   SwinvaModel(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_		)

        A__          =   ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths		) - 1))
        A__          =   int(config.embed_dim * 2 ** (len(config.depths		) - 1)		)

        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, expected_seq_len, expected_dim)		)


    def 						snake_case__     (      self :       Tuple,lowercase_ :       int,lowercase_ :       Dict,lowercase_ :       List[str]		)->    Optional[int]:





        '''simple docstring'''







        A__          =   SwinvaForMaskedImageModeling(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_		)
        self.parent.assertEqual(
            result.logits.shape,(self.batch_size, self.num_channels, self.image_size, self.image_size)		)

        # test greyscale images
        A__          =   1
        A__          =   SwinvaForMaskedImageModeling(lowercase_		)
        model.to(lowercase_		)
        model.eval()

        A__          =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]		)
        A__          =   model(lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, 1, self.image_size, self.image_size)		)


    def 						snake_case__     (      self :       List[Any],lowercase_ :       str,lowercase_ :       Tuple,lowercase_ :       Dict		)->    Tuple:





        '''simple docstring'''







        A__          =   self.type_sequence_label_size
        A__          =   SwinvaForImageClassification(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.type_sequence_label_size)		)


    def 						snake_case__     (      self :       int		)->    str:





        '''simple docstring'''







        A__          =   self.prepare_config_and_inputs()
        A__      ,						A__      ,						A__          =   config_and_inputs
        A__          =   {'pixel_values': pixel_values}
        return config, inputs_dict




@require_torch
class  A						(							_UpperCAmelCase    ,							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							(
        (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
    )
    lowerCamelCase			=							(
        {'feature-extraction': SwinvaModel, 'image-classification': SwinvaForImageClassification}
        if is_torch_available()
        else {}
    )

    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False


    def 						snake_case__     (      self :       Tuple		)->    Tuple:





        '''simple docstring'''







        A__          =   SwinvaModelTester(self		)
        A__          =   ConfigTester(self,config_class=lowercase_,embed_dim=3_7		)


    def 						snake_case__     (      self :       Dict		)->    Tuple:





        '''simple docstring'''







        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()


    def 						snake_case__     (      self :       int		)->    Optional[Any]:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowercase_		)


    @unittest.skip(reason='Got `CUDA error: misaligned address` with PyTorch 2.0.0.'		)
    def 						snake_case__     (      self :       int		)->    Optional[int]:





        '''simple docstring'''







        pass


    @unittest.skip(reason='Swinv2 does not use inputs_embeds'		)
    def 						snake_case__     (      self :       Any		)->    Dict:





        '''simple docstring'''







        pass


    def 						snake_case__     (      self :       int		)->    List[Any]:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            A__          =   model_class(lowercase_		)
            self.assertIsInstance(model.get_input_embeddings(),(nn.Module)		)
            A__          =   model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(lowercase_,nn.Linear		)		)


    def 						snake_case__     (      self :       Any		)->    int:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            A__          =   model_class(lowercase_		)
            A__          =   inspect.signature(model.forward		)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            A__          =   [*signature.parameters.keys()]

            A__          =   ['pixel_values']
            self.assertListEqual(arg_names[:1],lowercase_		)


    def 						snake_case__     (      self :       List[str]		)->    Tuple:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()
        A__          =   True

        for model_class in self.all_model_classes:
            A__          =   True
            A__          =   False
            A__          =   True
            A__          =   model_class(lowercase_		)
            model.to(lowercase_		)
            model.eval()
            with torch.no_grad():
                A__          =   model(**self._prepare_for_class(lowercase_,lowercase_		)		)
            A__          =   outputs.attentions
            A__          =   len(self.model_tester.depths		)
            self.assertEqual(len(lowercase_		),lowercase_		)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            A__          =   True
            A__          =   config.window_size**2
            A__          =   model_class(lowercase_		)
            model.to(lowercase_		)
            model.eval()
            with torch.no_grad():
                A__          =   model(**self._prepare_for_class(lowercase_,lowercase_		)		)
            A__          =   outputs.attentions
            self.assertEqual(len(lowercase_		),lowercase_		)

            self.assertListEqual(
                list(attentions[0].shape[-3:]		),[self.model_tester.num_heads[0], window_size_squared, window_size_squared],)
            A__          =   len(lowercase_		)

            # Check attention is always last and order is fine
            A__          =   True
            A__          =   True
            A__          =   model_class(lowercase_		)
            model.to(lowercase_		)
            model.eval()
            with torch.no_grad():
                A__          =   model(**self._prepare_for_class(lowercase_,lowercase_		)		)

            if hasattr(self.model_tester,'num_hidden_states_types'		):
                A__          =   self.model_tester.num_hidden_states_types
            else:
                # also another +1 for reshaped_hidden_states
                A__          =   2
            self.assertEqual(out_len + added_hidden_states,len(lowercase_		)		)

            A__          =   outputs.attentions

            self.assertEqual(len(lowercase_		),lowercase_		)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]		),[self.model_tester.num_heads[0], window_size_squared, window_size_squared],)


    def 						snake_case__     (      self :       Dict,lowercase_ :       Tuple,lowercase_ :       Optional[Any],lowercase_ :       Optional[int],lowercase_ :       Dict		)->    List[Any]:





        '''simple docstring'''







        A__          =   model_class(lowercase_		)
        model.to(lowercase_		)
        model.eval()

        with torch.no_grad():
            A__          =   model(**self._prepare_for_class(lowercase_,lowercase_		)		)

        A__          =   outputs.hidden_states

        A__          =   getattr(
            self.model_tester,'expected_num_hidden_layers',len(self.model_tester.depths		) + 1		)
        self.assertEqual(len(lowercase_		),lowercase_		)

        # Swinv2 has a different seq_length
        A__          =   (
            config.patch_size
            if isinstance(config.patch_size,collections.abc.Iterable		)
            else (config.patch_size, config.patch_size)
        )

        A__          =   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        self.assertListEqual(
            list(hidden_states[0].shape[-2:]		),[num_patches, self.model_tester.embed_dim],)

        A__          =   outputs.reshaped_hidden_states
        self.assertEqual(len(lowercase_		),lowercase_		)

        A__      ,						A__      ,						A__      ,						A__          =   reshaped_hidden_states[0].shape
        A__          =   (
            reshaped_hidden_states[0].view(lowercase_,lowercase_,height * width		).permute(0,2,1		)
        )
        self.assertListEqual(
            list(reshaped_hidden_states.shape[-2:]		),[num_patches, self.model_tester.embed_dim],)


    def 						snake_case__     (      self :       int		)->    List[Any]:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()

        A__          =   (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size,collections.abc.Iterable		)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )

        for model_class in self.all_model_classes:
            A__          =   True
            self.check_hidden_states_output(lowercase_,lowercase_,lowercase_,lowercase_		)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            A__          =   True

            self.check_hidden_states_output(lowercase_,lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    Dict:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()
        A__          =   3

        A__          =   (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size,collections.abc.Iterable		)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )
        A__          =   (
            config.patch_size
            if isinstance(config.patch_size,collections.abc.Iterable		)
            else (config.patch_size, config.patch_size)
        )

        A__          =   image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
        A__          =   image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

        for model_class in self.all_model_classes:
            A__          =   True
            self.check_hidden_states_output(lowercase_,lowercase_,lowercase_,(padded_height, padded_width)		)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            A__          =   True
            self.check_hidden_states_output(lowercase_,lowercase_,lowercase_,(padded_height, padded_width)		)


    def 						snake_case__     (      self :       Optional[Any]		)->    int:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*lowercase_		)


    def 						snake_case__     (      self :       Optional[int]		)->    Tuple:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*lowercase_		)


    @slow
    def 						snake_case__     (      self :       str		)->    Any:





        '''simple docstring'''







        for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            A__          =   SwinvaModel.from_pretrained(lowercase_		)
            self.assertIsNotNone(lowercase_		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    Tuple:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()

        A__          =   _config_zero_init(lowercase_		)
        for model_class in self.all_model_classes:
            A__          =   model_class(config=lowercase_		)
            for name, param in model.named_parameters():
                if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
                    self.assertIn(
                        ((param.data.mean() * 1E9).round() / 1E9).item(),[0.0, 1.0],msg=F'Parameter {name} of model {model_class} seems not properly initialized',)




@require_vision
@require_torch
class  A						(							unittest.TestCase ):







    """simple docstring"""





    @cached_property
    def 						snake_case__     (      self :       int		)->    Dict:





        '''simple docstring'''







        return (
            AutoImageProcessor.from_pretrained('microsoft/swinv2-tiny-patch4-window8-256'		)
            if is_vision_available()
            else None
        )


    @slow
    def 						snake_case__     (      self :       List[Any]		)->    Any:





        '''simple docstring'''







        A__          =   SwinvaForImageClassification.from_pretrained('microsoft/swinv2-tiny-patch4-window8-256'		).to(
            lowercase_		)
        A__          =   self.default_image_processor

        A__          =   Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'		)
        A__          =   image_processor(images=lowercase_,return_tensors='pt'		).to(lowercase_		)

        # forward pass
        with torch.no_grad():
            A__          =   model(**lowercase_		)

        # verify the logits
        A__          =   torch.Size((1, 1_0_0_0)		)
        self.assertEqual(outputs.logits.shape,lowercase_		)
        A__          =   torch.tensor([-0.3_947, -0.4_306, 0.0_026]		).to(lowercase_		)
        self.assertTrue(torch.allclose(outputs.logits[0, :3],lowercase_,atol=1E-4		)		)




import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
         GPTNeoXModel,
     )


class  A						:







    """simple docstring"""





    def __init__(      self :       str,lowercase_ :       Any,lowercase_ :       Tuple=1_3,lowercase_ :       str=7,lowercase_ :       Tuple=True,lowercase_ :       int=True,lowercase_ :       List[Any]=True,lowercase_ :       List[str]=True,lowercase_ :       List[str]=9_9,lowercase_ :       List[Any]=6_4,lowercase_ :       List[str]=5,lowercase_ :       Optional[Any]=4,lowercase_ :       Optional[Any]=3_7,lowercase_ :       Optional[Any]="gelu",lowercase_ :       int=0.1,lowercase_ :       str=0.1,lowercase_ :       Optional[Any]=5_1_2,lowercase_ :       int=1_6,lowercase_ :       List[Any]=2,lowercase_ :       Union[str, Any]=0.02,lowercase_ :       Tuple=3,lowercase_ :       List[Any]=4,lowercase_ :       str=None,)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   parent
        A__          =   batch_size
        A__          =   seq_length
        A__          =   is_training
        A__          =   use_input_mask
        A__          =   use_token_type_ids
        A__          =   use_labels
        A__          =   vocab_size
        A__          =   hidden_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   intermediate_size
        A__          =   hidden_act
        A__          =   hidden_dropout_prob
        A__          =   attention_probs_dropout_prob
        A__          =   max_position_embeddings
        A__          =   type_vocab_size
        A__          =   type_sequence_label_size
        A__          =   initializer_range
        A__          =   num_labels
        A__          =   num_choices
        A__          =   scope
        A__          =   vocab_size - 1


    def 						snake_case__     (      self :       str		)->    Optional[Any]:





        '''simple docstring'''







        A__          =   ids_tensor([self.batch_size, self.seq_length],self.vocab_size		)

        A__          =   None
        if self.use_input_mask:
            A__          =   random_attention_mask([self.batch_size, self.seq_length]		)

        A__          =   None
        if self.use_labels:
            A__          =   ids_tensor([self.batch_size, self.seq_length],self.num_labels		)

        A__          =   self.get_config()

        return config, input_ids, input_mask, token_labels


    def 						snake_case__     (      self :       List[Any]		)->    Tuple:





        '''simple docstring'''







        return GPTNeoXConfig(
            vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,is_decoder=lowercase_,initializer_range=self.initializer_range,pad_token_id=self.pad_token_id,)


    def 						snake_case__     (      self :       Optional[int]		)->    Union[str, Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.prepare_config_and_inputs()

        A__          =   True

        return config, input_ids, input_mask, token_labels


    def 						snake_case__     (      self :       Any,lowercase_ :       List[Any],lowercase_ :       List[Any],lowercase_ :       str		)->    Any:





        '''simple docstring'''







        A__          =   GPTNeoXModel(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        A__          =   model(lowercase_		)
        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)		)


    def 						snake_case__     (      self :       Union[str, Any],lowercase_ :       List[str],lowercase_ :       Dict,lowercase_ :       Optional[Any]		)->    Tuple:





        '''simple docstring'''







        A__          =   True
        A__          =   GPTNeoXModel(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)		)


    def 						snake_case__     (      self :       Union[str, Any],lowercase_ :       str,lowercase_ :       Union[str, Any],lowercase_ :       Union[str, Any],lowercase_ :       List[str]		)->    List[str]:





        '''simple docstring'''







        A__          =   GPTNeoXForCausalLM(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)		)


    def 						snake_case__     (      self :       Optional[int],lowercase_ :       Optional[int],lowercase_ :       Optional[int],lowercase_ :       Dict,lowercase_ :       Any		)->    int:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForQuestionAnswering(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)		)


    def 						snake_case__     (      self :       List[str],lowercase_ :       List[str],lowercase_ :       int,lowercase_ :       Union[str, Any],lowercase_ :       Optional[int]		)->    str:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForSequenceClassification(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   ids_tensor([self.batch_size],self.type_sequence_label_size		)
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)		)


    def 						snake_case__     (      self :       Any,lowercase_ :       Union[str, Any],lowercase_ :       List[Any],lowercase_ :       Optional[Any],lowercase_ :       int		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForTokenClassification(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)		)


    def 						snake_case__     (      self :       int,lowercase_ :       str,lowercase_ :       int,lowercase_ :       Union[str, Any]		)->    List[Any]:





        '''simple docstring'''







        A__          =   True
        A__          =   GPTNeoXForCausalLM(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()

        # first forward pass
        A__          =   model(lowercase_,attention_mask=lowercase_,use_cache=lowercase_		)
        A__          =   outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        A__          =   ids_tensor((self.batch_size, 3),config.vocab_size		)
        A__          =   ids_tensor((self.batch_size, 3),vocab_size=2		)

        # append to next input_ids and
        A__          =   torch.cat([input_ids, next_tokens],dim=-1		)
        A__          =   torch.cat([input_mask, next_mask],dim=-1		)

        A__          =   model(lowercase_,attention_mask=lowercase_,output_hidden_states=lowercase_		)
        A__          =   output_from_no_past['hidden_states'][0]
        A__          =   model(
            lowercase_,attention_mask=lowercase_,past_key_values=lowercase_,output_hidden_states=lowercase_,)['hidden_states'][0]

        # select random slice
        A__          =   ids_tensor((1,),output_from_past.shape[-1]		).item()
        A__          =   output_from_no_past[:, -3:, random_slice_idx].detach()
        A__          =   output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-3		)		)


    def 						snake_case__     (      self :       str		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   self.prepare_config_and_inputs()
        A__      ,						A__      ,						A__      ,						A__          =   config_and_inputs
        A__          =   {'input_ids': input_ids, 'attention_mask': input_mask}
        return config, inputs_dict




@require_torch
class  A						(							_UpperCAmelCase    ,							_UpperCAmelCase    ,							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							(
        (
            GPTNeoXModel,
            GPTNeoXForCausalLM,
            GPTNeoXForQuestionAnswering,
            GPTNeoXForSequenceClassification,
            GPTNeoXForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase			=							(GPTNeoXForCausalLM,) if is_torch_available() else ()
    lowerCamelCase			=							(
        {
            'feature-extraction': GPTNeoXModel,
            'question-answering': GPTNeoXForQuestionAnswering,
            'text-classification': GPTNeoXForSequenceClassification,
            'text-generation': GPTNeoXForCausalLM,
            'token-classification': GPTNeoXForTokenClassification,
            'zero-shot': GPTNeoXForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False


    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   GPTNeoXModelTester(self		)
        A__          =   ConfigTester(self,config_class=lowercase_,hidden_size=6_4,num_attention_heads=8		)


    def 						snake_case__     (      self :       Optional[Any]		)->    Union[str, Any]:





        '''simple docstring'''







        self.config_tester.run_common_tests()


    def 						snake_case__     (      self :       Union[str, Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Dict		)->    List[Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       List[str]		)->    Any:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_decoder()

        A__          =   None

        self.model_tester.create_and_check_model_as_decoder(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Optional[Any]		)->    str:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Dict		)->    Dict:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_causal_lm(*lowercase_		)


    def 						snake_case__     (      self :       Tuple		)->    List[Any]:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowercase_		)


    def 						snake_case__     (      self :       Any		)->    List[str]:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowercase_		)


    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowercase_		)


    @unittest.skip(reason='Feed forward chunking is not implemented'		)
    def 						snake_case__     (      self :       Union[str, Any]		)->    Optional[Any]:





        '''simple docstring'''







        pass


    @parameterized.expand([('linear',), ('dynamic',)]		)
    def 						snake_case__     (      self :       List[str],lowercase_ :       Any		)->    List[str]:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()
        A__          =   ids_tensor([1, 1_0],config.vocab_size		)
        A__          =   ids_tensor([1, int(config.max_position_embeddings * 1.5		)],config.vocab_size		)

        set_seed(4_2		)  # Fixed seed at init time so the two models get the same random weights
        A__          =   GPTNeoXModel(lowercase_		)
        original_model.to(lowercase_		)
        original_model.eval()
        A__          =   original_model(lowercase_		).last_hidden_state
        A__          =   original_model(lowercase_		).last_hidden_state

        set_seed(4_2		)  # Fixed seed at init time so the two models get the same random weights
        A__          =   {'type': scaling_type, 'factor': 10.0}
        A__          =   GPTNeoXModel(lowercase_		)
        scaled_model.to(lowercase_		)
        scaled_model.eval()
        A__          =   scaled_model(lowercase_		).last_hidden_state
        A__          =   scaled_model(lowercase_		).last_hidden_state

        # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
        # maximum sequence length, so the outputs for the short input should match.
        if scaling_type == "dynamic":
            self.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)
        else:
            self.assertFalse(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)

        # The output should be different for long inputs
        self.assertFalse(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)




@require_torch
class  A						(							unittest.TestCase ):







    """simple docstring"""





    @slow
    def 						snake_case__     (      self :       Tuple		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   AutoTokenizer.from_pretrained('EleutherAI/pythia-410m-deduped'		)
        for checkpointing in [True, False]:
            A__          =   GPTNeoXForCausalLM.from_pretrained('EleutherAI/pythia-410m-deduped'		)

            if checkpointing:
                model.gradient_checkpointing_enable()
            else:
                model.gradient_checkpointing_disable()
            model.to(lowercase_		)

            A__          =   tokenizer('My favorite food is',return_tensors='pt'		).to(lowercase_		)
            # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
            # See: https://github.com/huggingface/transformers/pull/24193
            A__          =   'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

            A__          =   model.generate(**lowercase_,do_sample=lowercase_,max_new_tokens=2_0		)
            A__          =   tokenizer.batch_decode(lowercase_		)[0]

            self.assertEqual(lowercase_,lowercase_		)

