

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_       		=  logging.get_logger(__name__)

lowercase_       		=  {
    "facebook/timesformer": "https://huggingface.co/facebook/timesformer/resolve/main/config.json",
}


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    lowerCamelCase			=							'timesformer'


    def __init__(      self :       List[Any],lowercase_ :       int=2_2_4,lowercase_ :       Optional[Any]=1_6,lowercase_ :       Dict=3,lowercase_ :       Optional[Any]=8,lowercase_ :       str=7_6_8,lowercase_ :       List[Any]=1_2,lowercase_ :       List[Any]=1_2,lowercase_ :       int=3_0_7_2,lowercase_ :       Tuple="gelu",lowercase_ :       int=0.0,lowercase_ :       Any=0.0,lowercase_ :       str=0.02,lowercase_ :       int=1E-6,lowercase_ :       List[str]=True,lowercase_ :       Any="divided_space_time",lowercase_ :       List[Any]=0,**lowercase_ :       Union[str, Any],)->    int:





        '''simple docstring'''







        super().__init__(**lowercase_		)

        A__          =   image_size
        A__          =   patch_size
        A__          =   num_channels
        A__          =   num_frames

        A__          =   hidden_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   intermediate_size
        A__          =   hidden_act
        A__          =   hidden_dropout_prob
        A__          =   attention_probs_dropout_prob
        A__          =   initializer_range
        A__          =   layer_norm_eps
        A__          =   qkv_bias

        A__          =   attention_type
        A__          =   drop_path_rate




import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         GPTNeoXForCausalLM,
         GPTNeoXForQuestionAnswering,
         GPTNeoXForSequenceClassification,
         GPTNeoXForTokenClassification,
         GPTNeoXModel,
     )


class  A						:







    """simple docstring"""





    def __init__(      self :       str,lowercase_ :       Any,lowercase_ :       Tuple=1_3,lowercase_ :       str=7,lowercase_ :       Tuple=True,lowercase_ :       int=True,lowercase_ :       List[Any]=True,lowercase_ :       List[str]=True,lowercase_ :       List[str]=9_9,lowercase_ :       List[Any]=6_4,lowercase_ :       List[str]=5,lowercase_ :       Optional[Any]=4,lowercase_ :       Optional[Any]=3_7,lowercase_ :       Optional[Any]="gelu",lowercase_ :       int=0.1,lowercase_ :       str=0.1,lowercase_ :       Optional[Any]=5_1_2,lowercase_ :       int=1_6,lowercase_ :       List[Any]=2,lowercase_ :       Union[str, Any]=0.02,lowercase_ :       Tuple=3,lowercase_ :       List[Any]=4,lowercase_ :       str=None,)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   parent
        A__          =   batch_size
        A__          =   seq_length
        A__          =   is_training
        A__          =   use_input_mask
        A__          =   use_token_type_ids
        A__          =   use_labels
        A__          =   vocab_size
        A__          =   hidden_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   intermediate_size
        A__          =   hidden_act
        A__          =   hidden_dropout_prob
        A__          =   attention_probs_dropout_prob
        A__          =   max_position_embeddings
        A__          =   type_vocab_size
        A__          =   type_sequence_label_size
        A__          =   initializer_range
        A__          =   num_labels
        A__          =   num_choices
        A__          =   scope
        A__          =   vocab_size - 1


    def 						snake_case__     (      self :       str		)->    Optional[Any]:





        '''simple docstring'''







        A__          =   ids_tensor([self.batch_size, self.seq_length],self.vocab_size		)

        A__          =   None
        if self.use_input_mask:
            A__          =   random_attention_mask([self.batch_size, self.seq_length]		)

        A__          =   None
        if self.use_labels:
            A__          =   ids_tensor([self.batch_size, self.seq_length],self.num_labels		)

        A__          =   self.get_config()

        return config, input_ids, input_mask, token_labels


    def 						snake_case__     (      self :       List[Any]		)->    Tuple:





        '''simple docstring'''







        return GPTNeoXConfig(
            vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,is_decoder=lowercase_,initializer_range=self.initializer_range,pad_token_id=self.pad_token_id,)


    def 						snake_case__     (      self :       Optional[int]		)->    Union[str, Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.prepare_config_and_inputs()

        A__          =   True

        return config, input_ids, input_mask, token_labels


    def 						snake_case__     (      self :       Any,lowercase_ :       List[Any],lowercase_ :       List[Any],lowercase_ :       str		)->    Any:





        '''simple docstring'''







        A__          =   GPTNeoXModel(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        A__          =   model(lowercase_		)
        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)		)


    def 						snake_case__     (      self :       Union[str, Any],lowercase_ :       List[str],lowercase_ :       Dict,lowercase_ :       Optional[Any]		)->    Tuple:





        '''simple docstring'''







        A__          =   True
        A__          =   GPTNeoXModel(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)		)


    def 						snake_case__     (      self :       Union[str, Any],lowercase_ :       str,lowercase_ :       Union[str, Any],lowercase_ :       Union[str, Any],lowercase_ :       List[str]		)->    List[str]:





        '''simple docstring'''







        A__          =   GPTNeoXForCausalLM(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)		)


    def 						snake_case__     (      self :       Optional[int],lowercase_ :       Optional[int],lowercase_ :       Optional[int],lowercase_ :       Dict,lowercase_ :       Any		)->    int:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForQuestionAnswering(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_		)
        self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)		)


    def 						snake_case__     (      self :       List[str],lowercase_ :       List[str],lowercase_ :       int,lowercase_ :       Union[str, Any],lowercase_ :       Optional[int]		)->    str:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForSequenceClassification(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   ids_tensor([self.batch_size],self.type_sequence_label_size		)
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)		)


    def 						snake_case__     (      self :       Any,lowercase_ :       Union[str, Any],lowercase_ :       List[Any],lowercase_ :       Optional[Any],lowercase_ :       int		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   self.num_labels
        A__          =   GPTNeoXForTokenClassification(lowercase_		)
        model.to(lowercase_		)
        model.eval()
        A__          =   model(lowercase_,attention_mask=lowercase_,labels=lowercase_		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)		)


    def 						snake_case__     (      self :       int,lowercase_ :       str,lowercase_ :       int,lowercase_ :       Union[str, Any]		)->    List[Any]:





        '''simple docstring'''







        A__          =   True
        A__          =   GPTNeoXForCausalLM(config=lowercase_		)
        model.to(lowercase_		)
        model.eval()

        # first forward pass
        A__          =   model(lowercase_,attention_mask=lowercase_,use_cache=lowercase_		)
        A__          =   outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        A__          =   ids_tensor((self.batch_size, 3),config.vocab_size		)
        A__          =   ids_tensor((self.batch_size, 3),vocab_size=2		)

        # append to next input_ids and
        A__          =   torch.cat([input_ids, next_tokens],dim=-1		)
        A__          =   torch.cat([input_mask, next_mask],dim=-1		)

        A__          =   model(lowercase_,attention_mask=lowercase_,output_hidden_states=lowercase_		)
        A__          =   output_from_no_past['hidden_states'][0]
        A__          =   model(
            lowercase_,attention_mask=lowercase_,past_key_values=lowercase_,output_hidden_states=lowercase_,)['hidden_states'][0]

        # select random slice
        A__          =   ids_tensor((1,),output_from_past.shape[-1]		).item()
        A__          =   output_from_no_past[:, -3:, random_slice_idx].detach()
        A__          =   output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-3		)		)


    def 						snake_case__     (      self :       str		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   self.prepare_config_and_inputs()
        A__      ,						A__      ,						A__      ,						A__          =   config_and_inputs
        A__          =   {'input_ids': input_ids, 'attention_mask': input_mask}
        return config, inputs_dict




@require_torch
class  A						(							_UpperCAmelCase    ,							_UpperCAmelCase    ,							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							(
        (
            GPTNeoXModel,
            GPTNeoXForCausalLM,
            GPTNeoXForQuestionAnswering,
            GPTNeoXForSequenceClassification,
            GPTNeoXForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase			=							(GPTNeoXForCausalLM,) if is_torch_available() else ()
    lowerCamelCase			=							(
        {
            'feature-extraction': GPTNeoXModel,
            'question-answering': GPTNeoXForQuestionAnswering,
            'text-classification': GPTNeoXForSequenceClassification,
            'text-generation': GPTNeoXForCausalLM,
            'token-classification': GPTNeoXForTokenClassification,
            'zero-shot': GPTNeoXForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False
    lowerCamelCase			=							False


    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   GPTNeoXModelTester(self		)
        A__          =   ConfigTester(self,config_class=lowercase_,hidden_size=6_4,num_attention_heads=8		)


    def 						snake_case__     (      self :       Optional[Any]		)->    Union[str, Any]:





        '''simple docstring'''







        self.config_tester.run_common_tests()


    def 						snake_case__     (      self :       Union[str, Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Dict		)->    List[Any]:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       List[str]		)->    Any:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_decoder()

        A__          =   None

        self.model_tester.create_and_check_model_as_decoder(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Optional[Any]		)->    str:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(lowercase_,lowercase_,lowercase_		)


    def 						snake_case__     (      self :       Dict		)->    Dict:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_causal_lm(*lowercase_		)


    def 						snake_case__     (      self :       Tuple		)->    List[Any]:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowercase_		)


    def 						snake_case__     (      self :       Any		)->    List[str]:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowercase_		)


    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowercase_		)


    @unittest.skip(reason='Feed forward chunking is not implemented'		)
    def 						snake_case__     (      self :       Union[str, Any]		)->    Optional[Any]:





        '''simple docstring'''







        pass


    @parameterized.expand([('linear',), ('dynamic',)]		)
    def 						snake_case__     (      self :       List[str],lowercase_ :       Any		)->    List[str]:





        '''simple docstring'''







        A__      ,						A__          =   self.model_tester.prepare_config_and_inputs_for_common()
        A__          =   ids_tensor([1, 1_0],config.vocab_size		)
        A__          =   ids_tensor([1, int(config.max_position_embeddings * 1.5		)],config.vocab_size		)

        set_seed(4_2		)  # Fixed seed at init time so the two models get the same random weights
        A__          =   GPTNeoXModel(lowercase_		)
        original_model.to(lowercase_		)
        original_model.eval()
        A__          =   original_model(lowercase_		).last_hidden_state
        A__          =   original_model(lowercase_		).last_hidden_state

        set_seed(4_2		)  # Fixed seed at init time so the two models get the same random weights
        A__          =   {'type': scaling_type, 'factor': 10.0}
        A__          =   GPTNeoXModel(lowercase_		)
        scaled_model.to(lowercase_		)
        scaled_model.eval()
        A__          =   scaled_model(lowercase_		).last_hidden_state
        A__          =   scaled_model(lowercase_		).last_hidden_state

        # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
        # maximum sequence length, so the outputs for the short input should match.
        if scaling_type == "dynamic":
            self.assertTrue(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)
        else:
            self.assertFalse(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)

        # The output should be different for long inputs
        self.assertFalse(torch.allclose(lowercase_,lowercase_,atol=1E-5		)		)




@require_torch
class  A						(							unittest.TestCase ):







    """simple docstring"""





    @slow
    def 						snake_case__     (      self :       Tuple		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   AutoTokenizer.from_pretrained('EleutherAI/pythia-410m-deduped'		)
        for checkpointing in [True, False]:
            A__          =   GPTNeoXForCausalLM.from_pretrained('EleutherAI/pythia-410m-deduped'		)

            if checkpointing:
                model.gradient_checkpointing_enable()
            else:
                model.gradient_checkpointing_disable()
            model.to(lowercase_		)

            A__          =   tokenizer('My favorite food is',return_tensors='pt'		).to(lowercase_		)
            # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
            # See: https://github.com/huggingface/transformers/pull/24193
            A__          =   'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

            A__          =   model.generate(**lowercase_,do_sample=lowercase_,max_new_tokens=2_0		)
            A__          =   tokenizer.batch_decode(lowercase_		)[0]

            self.assertEqual(lowercase_,lowercase_		)

