






from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.openai.modeling_tf_openai import (
					    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFOpenAIGPTDoubleHeadsModel,
					    TFOpenAIGPTForSequenceClassification,
					    TFOpenAIGPTLMHeadModel,
					    TFOpenAIGPTModel,
					)





class        a__      :






		def __init__(      self							, lowercase__							, )				->				Optional[Any]:
				__A    =							parent
				__A    =							13
				__A    =							7
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							99
				__A    =							32
				__A    =							2
				__A    =							4
				__A    =							37
				__A    =							"gelu"
				__A    =							0.1
				__A    =							0.1
				__A    =							512
				__A    =							16
				__A    =							2
				__A    =							0.02
				__A    =							3
				__A    =							4
				__A    =							None
				__A    =							self.vocab_size - 1






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size						)

				__A    =							None
				if self.use_input_mask:
						__A    =							random_attention_mask([self.batch_size, self.seq_length]						)

				__A    =							None
				if self.use_token_type_ids:
						__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size						)

				__A    =							None
				if self.use_mc_token_ids:
						__A    =							ids_tensor([self.batch_size, self.num_choices]							, self.seq_length						)

				__A    =							None
				__A    =							None
				__A    =							None
				if self.use_labels:
						__A    =							ids_tensor([self.batch_size]							, self.type_sequence_label_size						)
						__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.num_labels						)
						__A    =							ids_tensor([self.batch_size]							, self.num_choices						)

				__A    =							OpenAIGPTConfig(
				    vocab_size=self.vocab_size							, n_embd=self.hidden_size							, n_layer=self.num_hidden_layers							, n_head=self.num_attention_heads							, n_positions=self.max_position_embeddings							, pad_token_id=self.pad_token_id							, )

				__A    =							ids_tensor([self.num_hidden_layers, self.num_attention_heads]							, 2						)

				return (
				    config,
				    input_ids,
				    input_mask,
				    head_mask,
				    token_type_ids,
				    mc_token_ids,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Dict:
				__A    =							TFOpenAIGPTModel(config=lowercase__						)
				__A    =							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				__A    =							model(lowercase__						)

				__A    =							[input_ids, input_mask]
				__A    =							model(lowercase__						)

				__A    =							model(lowercase__						)

				self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.hidden_size)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Dict:
				__A    =							TFOpenAIGPTLMHeadModel(config=lowercase__						)
				__A    =							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				__A    =							model(lowercase__						)
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Optional[int]:
				__A    =							TFOpenAIGPTDoubleHeadsModel(config=lowercase__						)

				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)
				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)
				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)

				__A    =							{
				    "input_ids": multiple_choice_inputs_ids,
				    "mc_token_ids": mc_token_ids,
				    "attention_mask": multiple_choice_input_mask,
				    "token_type_ids": multiple_choice_token_type_ids,
				}
				__A    =							model(lowercase__						)
				self.parent.assertEqual(
				    result.logits.shape							, (self.batch_size, self.num_choices, self.seq_length, self.vocab_size)						)
				self.parent.assertEqual(result.mc_logits.shape							, (self.batch_size, self.num_choices)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Union[str, Any]:
				__A    =							self.num_labels
				__A    =							ids_tensor([self.batch_size]							, self.type_sequence_label_size						)
				__A    =							{
				    "input_ids": input_ids,
				    "attention_mask": input_mask,
				    "token_type_ids": token_type_ids,
				    "labels": sequence_labels,
				}
				__A    =							TFOpenAIGPTForSequenceClassification(lowercase__						)
				__A    =							model(lowercase__						)
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.num_labels)						)






		def 					_lowerCamelCase					(      self						)				->				str:
				__A    =							self.prepare_config_and_inputs()

				(
				    (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   
				)    =							config_and_inputs

				__A    =							{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
				return config, inputs_dict





@require_tf
class        a__      (	lowerCAmelCase__	,       lowerCAmelCase__	,       unittest.TestCase		):
		UpperCAmelCase_							:       Union[str, Any] 					= (
		    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
		    if is_tf_available()
		    else ()
		)
		UpperCAmelCase_							:       Optional[int] 					= (
		    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
		)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
		UpperCAmelCase_							:       List[Any] 					= (
		    {
		        'feature-extraction': TFOpenAIGPTModel,
		        'text-classification': TFOpenAIGPTForSequenceClassification,
		        'text-generation': TFOpenAIGPTLMHeadModel,
		        'zero-shot': TFOpenAIGPTForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		UpperCAmelCase_							:       List[Any] 					= False
		UpperCAmelCase_							:       List[Any] 					= False






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__						)				->				Dict:
				if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
						# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
						# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
						# tiny config could not be created.
						return True

				return False






		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							TFOpenAIGPTModelTester(self						)
				__A    =							ConfigTester(self							, config_class=lowercase__							, n_embd=37						)






		def 					_lowerCamelCase					(      self						)				->				List[str]:
				self.config_tester.run_common_tests()






		def 					_lowerCamelCase					(      self						)				->				str:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_model(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Optional[Any]:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_lm_head(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_double_head(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase__						)






		@slow
		def 					_lowerCamelCase					(      self						)				->				Optional[int]:
				for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__A    =							TFOpenAIGPTModel.from_pretrained(lowercase__						)
						self.assertIsNotNone(lowercase__						)





@require_tf
class        a__      (	unittest.TestCase		):






		@slow
		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							TFOpenAIGPTLMHeadModel.from_pretrained("openai-gpt"						)
				__A    =							tf.convert_to_tensor([[481, 4735, 544]]							, dtype=tf.intaa						)  # the president is
				__A    =							[
				    481,
				    4735,
				    544,
				    246,
				    963,
				    870,
				    762,
				    239,
				    244,
				    40477,
				    244,
				    249,
				    719,
				    881,
				    487,
				    544,
				    240,
				    244,
				    603,
				    481,
				]  # the president is a very good man. " \n " i\'m sure he is, " said the

				__A    =							model.generate(lowercase__							, do_sample=lowercase__						)
				self.assertListEqual(output_ids[0].numpy().tolist()							, lowercase__						)



import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


a__				:      Tuple    		=     [nn.LayerNorm]

a__				:      Tuple    		=     logging.get_logger(__name__)

a__				:      int    		=     version.parse(version.parse(torch.__version__).base_version)

a__				:      Dict    		=     parsed_torch_version_base >= version.parse('''2.0''')
a__				:      Any    		=     parsed_torch_version_base >= version.parse('''1.12''')
a__				:      Union[str, Any]    		=     parsed_torch_version_base >= version.parse('''1.11''')
a__				:      Union[str, Any]    		=     parsed_torch_version_base < version.parse('''1.11''')







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						a__ ,						self   ):



    """simple docstring"""



    from torch import _softmax_backward_data

    if is_torch_less_than_1_11:
        return _softmax_backward_data(a__ ,						a__ ,						parent.dim ,						self   )
    else:
        return _softmax_backward_data(a__ ,						a__ ,						parent.dim ,						self.dtype   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = 0   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = index.to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = layer.weight.index_select(a__ ,						a__   ).clone().detach()
    if layer.bias is not None:
        if dim == 1:
            SCREAMING_SNAKE_CASE				:				int				      = layer.bias.clone().detach()
        else:
            SCREAMING_SNAKE_CASE				:				List[Any]				      = layer.bias[index].clone().detach()
    SCREAMING_SNAKE_CASE				:				List[Any]				      = list(layer.weight.size()   )
    SCREAMING_SNAKE_CASE				:				List[Any]				      = len(a__   )
    SCREAMING_SNAKE_CASE				:				str				      = nn.Linear(new_size[1] ,						new_size[0] ,						bias=layer.bias is not None   ).to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = False
    new_layer.weight.copy_(W.contiguous()   )
    SCREAMING_SNAKE_CASE				:				Tuple				      = True
    if layer.bias is not None:
        SCREAMING_SNAKE_CASE				:				Dict				      = False
        new_layer.bias.copy_(b.contiguous()   )
        SCREAMING_SNAKE_CASE				:				Tuple				      = True
    return new_layer






class     a_       (			nn.Module							):



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase				,       _lowerCamelCase   )					->int:
           super().__init__()
           SCREAMING_SNAKE_CASE				:				List[str]				      = nf
           SCREAMING_SNAKE_CASE				:				List[Any]				      = nn.Parameter(torch.empty(_lowerCamelCase				,       _lowerCamelCase   )   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = nn.Parameter(torch.zeros(_lowerCamelCase   )   )
           nn.init.normal_(self.weight				,       std=0.0_2   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = x.size()[:-1] + (self.nf,)
           SCREAMING_SNAKE_CASE				:				int				      = torch.addmm(self.bias				,       x.view(-1				,       x.size(-1   )   )				,       self.weight   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = x.view(_lowerCamelCase   )
           return x







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = 1   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				str				      = index.to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Any				      = layer.weight.index_select(a__ ,						a__   ).clone().detach()
    if dim == 0:
        SCREAMING_SNAKE_CASE				:				Tuple				      = layer.bias.clone().detach()
    else:
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = layer.bias[index].clone().detach()
    SCREAMING_SNAKE_CASE				:				int				      = list(layer.weight.size()   )
    SCREAMING_SNAKE_CASE				:				Tuple				      = len(a__   )
    SCREAMING_SNAKE_CASE				:				Any				      = ConvaD(new_size[1] ,						new_size[0]   ).to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[int]				      = False
    new_layer.weight.copy_(W.contiguous()   )
    SCREAMING_SNAKE_CASE				:				int				      = True
    SCREAMING_SNAKE_CASE				:				Any				      = False
    new_layer.bias.copy_(b.contiguous()   )
    SCREAMING_SNAKE_CASE				:				int				      = True
    return new_layer







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = None   ):



    """simple docstring"""



    if isinstance(a__ ,						nn.Linear   ):
        return prune_linear_layer(a__ ,						a__ ,						dim=0 if dim is None else dim   )
    elif isinstance(a__ ,						a__   ):
        return prune_convad_layer(a__ ,						a__ ,						dim=1 if dim is None else dim   )
    else:
        raise ValueError(F"""Can't prune layer of class {layer.__class__}"""   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						*a__   ):



    """simple docstring"""



    assert len(a__   ) > 0, F"""{input_tensors} has to be a tuple/list of tensors"""

    # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
    SCREAMING_SNAKE_CASE				:				str				      = len(inspect.signature(a__   ).parameters   )
    if num_args_in_forward_chunk_fn != len(a__   ):
        raise ValueError(
            F"""forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(a__   )} input """
            '''tensors are given'''   )

    if chunk_size > 0:
        SCREAMING_SNAKE_CASE				:				Dict				      = input_tensors[0].shape[chunk_dim]
        for input_tensor in input_tensors:
            if input_tensor.shape[chunk_dim] != tensor_shape:
                raise ValueError(
                    F"""All input tenors have to be of the same shape: {tensor_shape}, """
                    F"""found shape {input_tensor.shape[chunk_dim]}"""   )

        if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
            raise ValueError(
                F"""The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk """
                F"""size {chunk_size}"""   )

        SCREAMING_SNAKE_CASE				:				Any				      = input_tensors[0].shape[chunk_dim] // chunk_size

        # chunk input tensor into tuples
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = tuple(input_tensor.chunk(a__ ,						dim=a__   ) for input_tensor in input_tensors   )
        # apply forward fn to every tuple
        SCREAMING_SNAKE_CASE				:				Tuple				      = tuple(forward_fn(*a__   ) for input_tensors_chunk in zip(*a__   )   )
        # concatenate output at same dimension
        return torch.cat(a__ ,						dim=a__   )

    return forward_fn(*a__   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						a__   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Optional[int]				      = torch.ones(a__ ,						a__   )
    SCREAMING_SNAKE_CASE				:				str				      = set(a__   ) - already_pruned_heads  # Convert to set and remove already pruned heads
    for head in heads:
        # Compute how many pruned heads are before the head and move the index accordingly
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = head - sum(1 if h < head else 0 for h in already_pruned_heads   )
        SCREAMING_SNAKE_CASE				:				str				      = 0
    SCREAMING_SNAKE_CASE				:				List[str]				      = mask.view(-1   ).contiguous().eq(1   )
    SCREAMING_SNAKE_CASE				:				torch.LongTensor				      = torch.arange(len(a__   )   )[mask].long()
    return heads, index







def 				UpperCAmelCase_(    *a__ ,						a__ = None   ):



    """simple docstring"""



    return torch.meshgrid(*a__ ,						indexing=a__   )







def 				UpperCAmelCase_(    a__   ):



    """simple docstring"""



    return tensor.device, storage_ptr(a__   ), storage_size(a__   )
