



from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers.models.ctrl.modeling_tf_ctrl import (
         TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCTRLForSequenceClassification,
         TFCTRLLMHeadModel,
         TFCTRLModel,
     )



class 		_lowercase		(							UpperCAmelCase__						):

   def __init__(   self      :							Optional[Any]				, __lowerCAmelCase      :							str				, )							->					Optional[Any]:


    """simple docstring"""






    a							=		parent
    a							=		13
    a							=		7
    a							=		True
    a							=		True
    a							=		True
    a							=		True
    a							=		True
    a							=		99
    a							=		32
    a							=		2
    a							=		4
    a							=		37
    a							=		"gelu"
    a							=		0.1
    a							=		0.1
    a							=		512
    a							=		16
    a							=		2
    a							=		0.0_2
    a							=		3
    a							=		4
    a							=		None
    a							=		self.vocab_size - 1

   def 		A				(   self      :							Tuple				)							->					Optional[int]:


    """simple docstring"""






    a							=		ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size				)

    a							=		None
    if self.use_input_mask:
     a							=		random_attention_mask([self.batch_size, self.seq_length]				)

    a							=		None
    if self.use_token_type_ids:
     a							=		ids_tensor([self.batch_size, self.seq_length]				, self.type_vocab_size				)

    a							=		None
    if self.use_mc_token_ids:
     a							=		ids_tensor([self.batch_size, self.num_choices]				, self.seq_length				)

    a							=		None
    a							=		None
    a							=		None
    if self.use_labels:
     a							=		ids_tensor([self.batch_size]				, self.type_sequence_label_size				)
     a							=		ids_tensor([self.batch_size, self.seq_length]				, self.num_labels				)
     a							=		ids_tensor([self.batch_size]				, self.num_choices				)

    a							=		CTRLConfig(
        vocab_size=self.vocab_size				, n_embd=self.hidden_size				, n_layer=self.num_hidden_layers				, n_head=self.num_attention_heads				, n_positions=self.max_position_embeddings				, pad_token_id=self.pad_token_id				, )

    a							=		ids_tensor([self.num_hidden_layers, self.num_attention_heads]				, 2				)

    return (
        config,
        input_ids,
        input_mask,
        head_mask,
        token_type_ids,
        mc_token_ids,
        sequence_labels,
        token_labels,
        choice_labels,
    )

   def 		A				(   self      :							List[str]				, __lowerCAmelCase      :							int				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							List[Any]				, *__lowerCAmelCase      :							Optional[int]				)							->					Dict:


    """simple docstring"""






    a							=		TFCTRLModel(config=__lowerCAmelCase				)
    a							=		{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
    a							=		model(__lowerCAmelCase				)

    a							=		[input_ids, None, input_mask]  # None is the input for 'past'
    a							=		model(__lowerCAmelCase				)

    a							=		model(__lowerCAmelCase				)

    self.parent.assertEqual(result.last_hidden_state.shape				, (self.batch_size, self.seq_length, self.hidden_size)				)

   def 		A				(   self      :							Optional[Any]				, __lowerCAmelCase      :							int				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							Dict				, __lowerCAmelCase      :							List[str]				, __lowerCAmelCase      :							Optional[int]				, *__lowerCAmelCase      :							List[str]				)							->					Optional[Any]:


    """simple docstring"""






    a							=		TFCTRLLMHeadModel(config=__lowerCAmelCase				)
    a							=		{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
    a							=		model(__lowerCAmelCase				)
    self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.seq_length, self.vocab_size)				)

   def 		A				(   self      :							int				, __lowerCAmelCase      :							List[Any]				, __lowerCAmelCase      :							Optional[int]				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							List[Any]				, __lowerCAmelCase      :							Dict				, *__lowerCAmelCase      :							Optional[int]				)							->					Tuple:


    """simple docstring"""






    a							=		self.num_labels
    a							=		ids_tensor([self.batch_size]				, self.type_sequence_label_size				)
    a							=		{
        "input_ids": input_ids,
        "token_type_ids": token_type_ids,
        "labels": sequence_labels,
    }
    a							=		TFCTRLForSequenceClassification(__lowerCAmelCase				)
    a							=		model(__lowerCAmelCase				)
    self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.num_labels)				)




   def 		A				(   self      :							Union[str, Any]				)							->					List[str]:


    """simple docstring"""






    a							=		self.prepare_config_and_inputs()

    (
        (
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	(
        a
    )		,	
    )							=		config_and_inputs

    a							=		{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
    return config, inputs_dict



@require_tf
class 		_lowercase		(							UpperCAmelCase__, UpperCAmelCase__, unittest.TestCase						):
   _UpperCAmelCase  						=      (TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
   _UpperCAmelCase  						=      (TFCTRLLMHeadModel,) if is_tf_available() else ()
   _UpperCAmelCase  						=      (
       {
           '''feature-extraction''': TFCTRLModel,
           '''text-classification''': TFCTRLForSequenceClassification,
           '''text-generation''': TFCTRLLMHeadModel,
           '''zero-shot''': TFCTRLForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      False

   def 		A				(   self      :							Union[str, Any]				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							Optional[Any]				, __lowerCAmelCase      :							int				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							List[str]				)							->					Any:


    """simple docstring"""






    if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
     # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
     # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
     # config could not be created.
     return True

    return False

   def 		A				(   self      :							str				)							->					Tuple:


    """simple docstring"""






    a							=		TFCTRLModelTester(self				)
    a							=		ConfigTester(self				, config_class=__lowerCAmelCase				, n_embd=37				)

   def 		A				(   self      :							Dict				)							->					List[str]:


    """simple docstring"""






    self.config_tester.run_common_tests()

   def 		A				(   self      :							Union[str, Any]				)							->					str:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_ctrl_model(*__lowerCAmelCase				)

   def 		A				(   self      :							Optional[int]				)							->					Tuple:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_ctrl_lm_head(*__lowerCAmelCase				)

   def 		A				(   self      :							List[Any]				)							->					Optional[Any]:


    """simple docstring"""






    a							=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_ctrl_for_sequence_classification(*__lowerCAmelCase				)

   def 		A				(   self      :							Any				)							->					Tuple:


    """simple docstring"""






    a		,	a							=		self.model_tester.prepare_config_and_inputs_for_common()
    a							=		[TFCTRLLMHeadModel]
    a							=		[TFCTRLForSequenceClassification]

    for model_class in self.all_model_classes:
     a							=		model_class(__lowerCAmelCase				)
     model.build()  # may be needed for the get_bias() call below
     assert isinstance(model.get_input_embeddings()				, tf.keras.layers.Layer				)

     if model_class in list_lm_models:
      a							=		model.get_output_embeddings()
      assert isinstance(__lowerCAmelCase				, tf.keras.layers.Layer				)
      a							=		model.get_bias()
      assert isinstance(__lowerCAmelCase				, __lowerCAmelCase				)
      for k, v in name.items():
       assert isinstance(__lowerCAmelCase				, tf.Variable				)
     elif model_class in list_other_models_with_output_ebd:
      a							=		model.get_output_embeddings()
      assert isinstance(__lowerCAmelCase				, tf.keras.layers.Layer				)
      a							=		model.get_bias()
      assert name is None
     else:
      a							=		model.get_output_embeddings()
      assert x is None
      a							=		model.get_bias()
      assert name is None




   @slow
   def 		A				(   self      :							Union[str, Any]				)							->					Tuple:


    """simple docstring"""






    for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     a							=		TFCTRLModel.from_pretrained(__lowerCAmelCase				)
     self.assertIsNotNone(__lowerCAmelCase				)



@require_tf
class 		_lowercase		(							unittest.TestCase						):

   @slow
   def 		A				(   self      :							Optional[int]				)							->					Dict:


    """simple docstring"""






    a							=		TFCTRLLMHeadModel.from_pretrained("ctrl"				)
    a							=		tf.convert_to_tensor([[1_1859, 0, 1611, 8]]				, dtype=tf.intaa				)  # Legal the president is
    a							=		[
        1_1859,
        0,
        1611,
        8,
        5,
        150,
        2_6449,
        2,
        19,
        348,
        469,
        3,
        2595,
        48,
        2_0740,
        24_6533,
        24_6533,
        19,
        30,
        5,
    ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

    a							=		model.generate(__lowerCAmelCase				, do_sample=__lowerCAmelCase				)
    self.assertListEqual(output_ids[0].numpy().tolist()				, __lowerCAmelCase				)



import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


a__				:      Tuple    		=     [nn.LayerNorm]

a__				:      Tuple    		=     logging.get_logger(__name__)

a__				:      int    		=     version.parse(version.parse(torch.__version__).base_version)

a__				:      Dict    		=     parsed_torch_version_base >= version.parse('''2.0''')
a__				:      Any    		=     parsed_torch_version_base >= version.parse('''1.12''')
a__				:      Union[str, Any]    		=     parsed_torch_version_base >= version.parse('''1.11''')
a__				:      Union[str, Any]    		=     parsed_torch_version_base < version.parse('''1.11''')







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						a__ ,						self   ):



    """simple docstring"""



    from torch import _softmax_backward_data

    if is_torch_less_than_1_11:
        return _softmax_backward_data(a__ ,						a__ ,						parent.dim ,						self   )
    else:
        return _softmax_backward_data(a__ ,						a__ ,						parent.dim ,						self.dtype   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = 0   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = index.to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = layer.weight.index_select(a__ ,						a__   ).clone().detach()
    if layer.bias is not None:
        if dim == 1:
            SCREAMING_SNAKE_CASE				:				int				      = layer.bias.clone().detach()
        else:
            SCREAMING_SNAKE_CASE				:				List[Any]				      = layer.bias[index].clone().detach()
    SCREAMING_SNAKE_CASE				:				List[Any]				      = list(layer.weight.size()   )
    SCREAMING_SNAKE_CASE				:				List[Any]				      = len(a__   )
    SCREAMING_SNAKE_CASE				:				str				      = nn.Linear(new_size[1] ,						new_size[0] ,						bias=layer.bias is not None   ).to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[Any]				      = False
    new_layer.weight.copy_(W.contiguous()   )
    SCREAMING_SNAKE_CASE				:				Tuple				      = True
    if layer.bias is not None:
        SCREAMING_SNAKE_CASE				:				Dict				      = False
        new_layer.bias.copy_(b.contiguous()   )
        SCREAMING_SNAKE_CASE				:				Tuple				      = True
    return new_layer






class     a_       (			nn.Module							):



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase				,       _lowerCamelCase   )					->int:
           super().__init__()
           SCREAMING_SNAKE_CASE				:				List[str]				      = nf
           SCREAMING_SNAKE_CASE				:				List[Any]				      = nn.Parameter(torch.empty(_lowerCamelCase				,       _lowerCamelCase   )   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = nn.Parameter(torch.zeros(_lowerCamelCase   )   )
           nn.init.normal_(self.weight				,       std=0.0_2   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = x.size()[:-1] + (self.nf,)
           SCREAMING_SNAKE_CASE				:				int				      = torch.addmm(self.bias				,       x.view(-1				,       x.size(-1   )   )				,       self.weight   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = x.view(_lowerCamelCase   )
           return x







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = 1   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				str				      = index.to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Any				      = layer.weight.index_select(a__ ,						a__   ).clone().detach()
    if dim == 0:
        SCREAMING_SNAKE_CASE				:				Tuple				      = layer.bias.clone().detach()
    else:
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = layer.bias[index].clone().detach()
    SCREAMING_SNAKE_CASE				:				int				      = list(layer.weight.size()   )
    SCREAMING_SNAKE_CASE				:				Tuple				      = len(a__   )
    SCREAMING_SNAKE_CASE				:				Any				      = ConvaD(new_size[1] ,						new_size[0]   ).to(layer.weight.device   )
    SCREAMING_SNAKE_CASE				:				Optional[int]				      = False
    new_layer.weight.copy_(W.contiguous()   )
    SCREAMING_SNAKE_CASE				:				int				      = True
    SCREAMING_SNAKE_CASE				:				Any				      = False
    new_layer.bias.copy_(b.contiguous()   )
    SCREAMING_SNAKE_CASE				:				int				      = True
    return new_layer







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ = None   ):



    """simple docstring"""



    if isinstance(a__ ,						nn.Linear   ):
        return prune_linear_layer(a__ ,						a__ ,						dim=0 if dim is None else dim   )
    elif isinstance(a__ ,						a__   ):
        return prune_convad_layer(a__ ,						a__ ,						dim=1 if dim is None else dim   )
    else:
        raise ValueError(F"""Can't prune layer of class {layer.__class__}"""   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						*a__   ):



    """simple docstring"""



    assert len(a__   ) > 0, F"""{input_tensors} has to be a tuple/list of tensors"""

    # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
    SCREAMING_SNAKE_CASE				:				str				      = len(inspect.signature(a__   ).parameters   )
    if num_args_in_forward_chunk_fn != len(a__   ):
        raise ValueError(
            F"""forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(a__   )} input """
            '''tensors are given'''   )

    if chunk_size > 0:
        SCREAMING_SNAKE_CASE				:				Dict				      = input_tensors[0].shape[chunk_dim]
        for input_tensor in input_tensors:
            if input_tensor.shape[chunk_dim] != tensor_shape:
                raise ValueError(
                    F"""All input tenors have to be of the same shape: {tensor_shape}, """
                    F"""found shape {input_tensor.shape[chunk_dim]}"""   )

        if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
            raise ValueError(
                F"""The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk """
                F"""size {chunk_size}"""   )

        SCREAMING_SNAKE_CASE				:				Any				      = input_tensors[0].shape[chunk_dim] // chunk_size

        # chunk input tensor into tuples
        SCREAMING_SNAKE_CASE				:				Optional[int]				      = tuple(input_tensor.chunk(a__ ,						dim=a__   ) for input_tensor in input_tensors   )
        # apply forward fn to every tuple
        SCREAMING_SNAKE_CASE				:				Tuple				      = tuple(forward_fn(*a__   ) for input_tensors_chunk in zip(*a__   )   )
        # concatenate output at same dimension
        return torch.cat(a__ ,						dim=a__   )

    return forward_fn(*a__   )







def 				UpperCAmelCase_(    a__ ,						a__ ,						a__ ,						a__   ):



    """simple docstring"""



    SCREAMING_SNAKE_CASE				:				Optional[int]				      = torch.ones(a__ ,						a__   )
    SCREAMING_SNAKE_CASE				:				str				      = set(a__   ) - already_pruned_heads  # Convert to set and remove already pruned heads
    for head in heads:
        # Compute how many pruned heads are before the head and move the index accordingly
        SCREAMING_SNAKE_CASE				:				Optional[Any]				      = head - sum(1 if h < head else 0 for h in already_pruned_heads   )
        SCREAMING_SNAKE_CASE				:				str				      = 0
    SCREAMING_SNAKE_CASE				:				List[str]				      = mask.view(-1   ).contiguous().eq(1   )
    SCREAMING_SNAKE_CASE				:				torch.LongTensor				      = torch.arange(len(a__   )   )[mask].long()
    return heads, index







def 				UpperCAmelCase_(    *a__ ,						a__ = None   ):



    """simple docstring"""



    return torch.meshgrid(*a__ ,						indexing=a__   )







def 				UpperCAmelCase_(    a__   ):



    """simple docstring"""



    return tensor.device, storage_ptr(a__   ), storage_size(a__   )
