






"""simple docstring"""


import os
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

lowercase_								=	logging.get_logger(__name__)

lowercase_								=	{'vocab_file': 'tokenizer.model'}

lowercase_								=	{
    'vocab_file': {
        'hf-internal-testing/llama-tokenizer': 'https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model',
    },
    'tokenizer_file': {
        'hf-internal-testing/llama-tokenizer': 'https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json',
    },
}
lowercase_								=	{
    'hf-internal-testing/llama-tokenizer': 2048,
}
lowercase_								=	'▁'

lowercase_	,						lowercase_								=	'[INST]', '[/INST]'
lowercase_	,						lowercase_								=	'<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
lowercase_								=	'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'


class    snake_case						(				_lowerCAmelCase		):


   '''simple docstring'''

   A_		:      List[Any]							=					VOCAB_FILES_NAMES
   A_		:      List[Any]							=					PRETRAINED_VOCAB_FILES_MAP
   A_		:      Union[str, Any]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A_		:      Tuple							=					["input_ids", "attention_mask"]


   def __init__(      self					:	str,			_lowerCamelCase					:	int,			_lowerCamelCase					:	Tuple="<unk>",			_lowerCamelCase					:	Dict="<s>",			_lowerCamelCase					:	int="</s>",			_lowerCamelCase					:	Tuple=None,			_lowerCamelCase					:	Optional[Dict[str, Any]] = None,			_lowerCamelCase					:	List[Any]=True,			_lowerCamelCase					:	str=False,			_lowerCamelCase					:	Any=False,			_lowerCamelCase					:	str=True,			**_lowerCamelCase					:	Dict,			):

     '''simple docstring'''
     __A		     =					{} if sp_model_kwargs is None else sp_model_kwargs
     __A		     =					AddedToken(_lowerCamelCase,			lstrip=_lowerCamelCase,			rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase,			_lowerCamelCase    ) else bos_token
     __A		     =					AddedToken(_lowerCamelCase,			lstrip=_lowerCamelCase,			rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase,			_lowerCamelCase    ) else eos_token
     __A		     =					AddedToken(_lowerCamelCase,			lstrip=_lowerCamelCase,			rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase,			_lowerCamelCase    ) else unk_token
     __A		     =					AddedToken(_lowerCamelCase,			lstrip=_lowerCamelCase,			rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase,			_lowerCamelCase    ) else pad_token
     super().__init__(
         bos_token=_lowerCamelCase,			eos_token=_lowerCamelCase,			unk_token=_lowerCamelCase,			pad_token=_lowerCamelCase,			add_bos_token=_lowerCamelCase,			add_eos_token=_lowerCamelCase,			sp_model_kwargs=self.sp_model_kwargs,			clean_up_tokenization_spaces=_lowerCamelCase,			legacy=_lowerCamelCase,			**_lowerCamelCase,			)
     if legacy:
       logger.warning_once(
           f'You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'
           ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''    )
     __A		     =					legacy
     __A		     =					vocab_file
     __A		     =					add_bos_token
     __A		     =					add_eos_token
     __A		     =					spm.SentencePieceProcessor(**self.sp_model_kwargs    )
     self.sp_model.Load(_lowerCamelCase    )


   def __getstate__(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					self.__dict__.copy()
     __A		     =					None
     __A		     =					self.sp_model.serialized_model_proto()
     return state


   def __setstate__(      self					:	Tuple,			_lowerCamelCase					:	List[Any]    ):

     '''simple docstring'''
     __A		     =					d
     __A		     =					spm.SentencePieceProcessor(**self.sp_model_kwargs    )
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )


   @property
   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     return self.sp_model.get_piece_size()


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					{self.convert_ids_to_tokens(_lowerCamelCase    ): i for i in range(self.vocab_size    )}
     vocab.update(self.added_tokens_encoder    )
     return vocab


   def 							_SCREAMING_SNAKE_CASE							(      self					:	int,			_lowerCamelCase					:	Dict,			**_lowerCamelCase					:	Any    ):

     '''simple docstring'''
     # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
     # the beginning of the text
     if not self.legacy:
       __A		     =					SPIECE_UNDERLINE + text.replace(_lowerCamelCase,			''' '''    )
     return super().tokenize(_lowerCamelCase,			**_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	Any    ):

     '''simple docstring'''
     if not self.legacy:
       __A		     =					text.startswith(_lowerCamelCase    )
       if is_first:
         __A		     =					text[1:]

     __A		     =					self.sp_model.encode(_lowerCamelCase,			out_type=_lowerCamelCase    )

     if not self.legacy and not is_first and not text.startswith(''' '''    ) and tokens[0].startswith(_lowerCamelCase    ):
       __A		     =					([tokens[0][1:]] if len(tokens[0]    ) > 1 else []) + tokens[1:]
     return tokens


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	List[Any]    ):

     '''simple docstring'''
     return self.sp_model.piece_to_id(_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int],			_lowerCamelCase					:	Tuple    ):

     '''simple docstring'''
     __A		     =					self.sp_model.IdToPiece(_lowerCamelCase    )
     return token


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					[]
     __A		     =					''''''
     __A		     =					False
     for i, token in enumerate(_lowerCamelCase    ):
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
         if not prev_is_special and i != 0:
           out_string += " "
         out_string += self.sp_model.decode(_lowerCamelCase    ) + token
         __A		     =					True
         __A		     =					[]
       else:
         current_sub_tokens.append(_lowerCamelCase    )
         __A		     =					False
     out_string += self.sp_model.decode(_lowerCamelCase    )
     return out_string


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	str,			_lowerCamelCase					:	Optional[str] = None    ):

     '''simple docstring'''
     if not os.path.isdir(_lowerCamelCase    ):
       logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
       return
     __A		     =					os.path.join(
         _lowerCamelCase,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

     if os.path.abspath(self.vocab_file    ) != os.path.abspath(_lowerCamelCase    ) and os.path.isfile(self.vocab_file    ):
       copyfile(self.vocab_file,			_lowerCamelCase    )
     elif not os.path.isfile(self.vocab_file    ):
       with open(_lowerCamelCase,			'''wb'''    ) as fi:
         __A		     =					self.sp_model.serialized_model_proto()
         fi.write(_lowerCamelCase    )

     return (out_vocab_file,)


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	str=None    ):

     '''simple docstring'''
     __A		     =					[self.bos_token_id] if self.add_bos_token else []
     __A		     =					[self.eos_token_id] if self.add_eos_token else []

     __A		     =					bos_token_id + token_ids_a + eos_token_id

     if token_ids_a is not None:
       __A		     =					output + bos_token_id + token_ids_a + eos_token_id

     return output


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Union[str, Any],			_lowerCamelCase					:	List[int],			_lowerCamelCase					:	Optional[List[int]] = None,			_lowerCamelCase					:	bool = False    ):

     '''simple docstring'''
     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowerCamelCase,			token_ids_a=_lowerCamelCase,			already_has_special_tokens=_lowerCamelCase    )

     __A		     =					[1] if self.add_bos_token else []
     __A		     =					[1] if self.add_eos_token else []

     if token_ids_a is None:
       return bos_token_id + ([0] * len(_lowerCamelCase    )) + eos_token_id
     return (
         bos_token_id
         + ([0] * len(_lowerCamelCase    ))
         + eos_token_id
         + bos_token_id
         + ([0] * len(_lowerCamelCase    ))
         + eos_token_id
     )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	List[int],			_lowerCamelCase					:	Optional[List[int]] = None    ):

     '''simple docstring'''
     __A		     =					[self.bos_token_id] if self.add_bos_token else []
     __A		     =					[self.eos_token_id] if self.add_eos_token else []

     __A		     =					[0] * len(bos_token_id + token_ids_a + eos_token_id    )

     if token_ids_a is not None:
       output += [1] * len(bos_token_id + token_ids_a + eos_token_id    )

     return output






   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any,			_lowerCamelCase					:	"Conversation"    ):

     '''simple docstring'''
     __A		     =					list(conversation.iter_texts()    )
     if not all([is_user for is_user, msg in dialogue[::2]]    ) or not all(
         [not is_user for is_user, msg in dialogue[1::2]]    ):
       raise ValueError(
           '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''    )

     __A		     =					[]
     if len(conversation.past_user_inputs    ) > 0:
       if not conversation.past_user_inputs[0].startswith(_lowerCamelCase    ) or E_SYS not in conversation.past_user_inputs[0]:
         __A		     =					(
             B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
         )
     elif not dialogue[0][1].startswith(_lowerCamelCase    ) or E_SYS not in dialogue[0][1]:
       __A		     =					(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

     dialog_tokens += sum(
         [
             [self.bos_token_id]
             + self.encode(
                 f'{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} ',			add_special_tokens=_lowerCamelCase    )
             + [self.eos_token_id]
             for prompt, answer in zip(dialogue[::2],			dialogue[1::2]    )
         ],			[],			)
     if not (dialogue[-1][0]):
       raise ValueError(f'Last message must be from user, got {dialogue[-1]["role"]}'    )
     dialog_tokens += [self.bos_token_id] + self.encode(
         f'{B_INST} {(dialogue[-1][1]).strip()} {E_INST}',			add_special_tokens=_lowerCamelCase    )
     return dialog_tokens












"""simple docstring"""


from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.gptj.modeling_tf_gptj import (
           TFGPTJForCausalLM,
           TFGPTJForQuestionAnswering,
           TFGPTJForSequenceClassification,
           TFGPTJModel,
           shape_list,
       )


class    snake_case						:


   '''simple docstring'''

   def __init__(      self					:	List[Any],			_lowerCamelCase					:	Optional[int]    ):

     '''simple docstring'''
     __A		     =					parent
     __A		     =					13
     __A		     =					7
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					99
     __A		     =					32
     __A		     =					4
     __A		     =					2
     __A		     =					4
     __A		     =					37
     __A		     =					'''gelu'''
     __A		     =					0.1
     __A		     =					0.1
     __A		     =					5_12
     __A		     =					16
     __A		     =					2
     __A		     =					0.02
     __A		     =					3
     __A		     =					4
     __A		     =					None
     __A		     =					self.vocab_size - 1
     __A		     =					self.vocab_size - 1
     __A		     =					self.vocab_size - 1


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.vocab_size    )

     __A		     =					None
     if self.use_input_mask:
       __A		     =					random_attention_mask([self.batch_size, self.seq_length]    )

     __A		     =					None
     if self.use_token_type_ids:
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size    )

     __A		     =					None
     if self.use_mc_token_ids:
       __A		     =					ids_tensor([self.batch_size, self.num_choices],			self.seq_length    )

     __A		     =					None
     __A		     =					None
     __A		     =					None
     if self.use_labels:
       __A		     =					ids_tensor([self.batch_size],			self.type_sequence_label_size    )
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.num_labels    )
       __A		     =					ids_tensor([self.batch_size],			self.num_choices    )

     __A		     =					GPTJConfig(
         vocab_size=self.vocab_size,			n_embd=self.hidden_size,			n_layer=self.num_hidden_layers,			n_head=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			n_positions=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			initializer_range=self.initializer_range,			bos_token_id=self.bos_token_id,			eos_token_id=self.eos_token_id,			pad_token_id=self.pad_token_id,			rotary_dim=self.rotary_dim,			return_dict=_lowerCamelCase,			)

     __A		     =					ids_tensor([self.num_hidden_layers, self.num_attention_heads],			2    )

     return (
         config,
         input_ids,
         input_mask,
         head_mask,
         token_type_ids,
         mc_token_ids,
         sequence_labels,
         token_labels,
         choice_labels,
     )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	int,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	List[str],			*_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )
     __A		     =					{
         '''input_ids''': input_ids,
         '''attention_mask''': input_mask,
         '''token_type_ids''': token_type_ids,
     }
     __A		     =					model(_lowerCamelCase    )

     __A		     =					[input_ids, None, input_mask]  # None is the input for 'past'
     __A		     =					model(_lowerCamelCase    )

     __A		     =					model(_lowerCamelCase    )

     self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	int,			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Optional[int],			*_lowerCamelCase					:	List[str]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     # first forward pass
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase    )
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )

     self.parent.assertTrue(len(_lowerCamelCase    ) == len(_lowerCamelCase    )    )
     self.parent.assertTrue(len(_lowerCamelCase    ) == len(_lowerCamelCase    ) + 1    )

     __A     ,       __A		     =					outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 1),			config.vocab_size    )
     __A		     =					ids_tensor([self.batch_size, 1],			self.type_vocab_size    )

     # append to next input_ids and token_type_ids
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([token_type_ids, next_token_types],			axis=-1    )

     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			past_key_values=_lowerCamelCase    )[
         '''last_hidden_state'''
     ]

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -1, random_slice_idx]
     __A		     =					output_from_past[:, 0, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-6    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	int,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Dict,			*_lowerCamelCase					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     # create attention mask
     __A		     =					self.seq_length // 2
     __A		     =					tf.ones((self.batch_size, half_seq_length),			dtype=tf.intaa    )
     __A		     =					tf.zeros((self.batch_size, self.seq_length - half_seq_length),			dtype=tf.intaa    )
     __A		     =					tf.concat([attn_mask_begin, attn_mask_end],			axis=1    )

     # first forward pass
     __A     ,       __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase    ).to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 1),			config.vocab_size    )

     # change a random masked slice from input_ids
     __A		     =					ids_tensor((1,),			_lowerCamelCase    ).numpy() + 1
     __A		     =					ids_tensor((self.batch_size, self.seq_length),			config.vocab_size    )
     __A		     =					tf.range(self.seq_length    ) == (self.seq_length - random_seq_idx_to_change)
     __A		     =					tf.transpose(
         tf.broadcast_to(tf.expand_dims(_lowerCamelCase,			-1    ),			(self.seq_length, self.batch_size)    )    )
     __A		     =					tf.where(_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase    )

     # append to next input_ids and attn_mask
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([attn_mask, tf.ones((shape_list(_lowerCamelCase    )[0], 1),			dtype=tf.intaa    )],			axis=1    )

     # get two different outputs
     __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(_lowerCamelCase,			past_key_values=_lowerCamelCase,			attention_mask=_lowerCamelCase    )[
         '''last_hidden_state'''
     ]

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -1, random_slice_idx]
     __A		     =					output_from_past[:, 0, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-12    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	int,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Tuple,			*_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     __A		     =					input_ids[:1, :]
     __A		     =					input_mask[:1, :]
     __A		     =					token_type_ids[:1, :]
     __A		     =					1

     # first forward pass
     __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )

     __A     ,       __A		     =					outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 3),			config.vocab_size    )
     __A		     =					ids_tensor((self.batch_size, 3),			2    )
     __A		     =					ids_tensor((self.batch_size, 3),			self.type_vocab_size    )

     # append to next input_ids and token_type_ids
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([input_mask, next_attn_mask],			axis=-1    )
     __A		     =					tf.concat([token_type_ids, next_token_types],			axis=-1    )

     __A		     =					model(
         _lowerCamelCase,			token_type_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(
         _lowerCamelCase,			token_type_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase,			past_key_values=_lowerCamelCase,			)['''last_hidden_state''']
     self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]    )

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -3:, random_slice_idx]
     __A		     =					output_from_past[:, :, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-3    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Tuple,			*_lowerCamelCase					:	int    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM(config=_lowerCamelCase    )
     __A		     =					{
         '''input_ids''': input_ids,
         '''attention_mask''': input_mask,
         '''token_type_ids''': token_type_ids,
     }
     __A		     =					model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.prepare_config_and_inputs()

     (
         (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       
     )		     =					config_and_inputs

     __A		     =					{
         '''input_ids''': input_ids,
         '''token_type_ids''': token_type_ids,
         '''attention_mask''': input_mask,
     }
     return config, inputs_dict





@require_tf
class    snake_case						(				_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      unittest.TestCase		):


   '''simple docstring'''

   A_		:      Union[str, Any]							=					(
       (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
       if is_tf_available()
       else ()
   )

   A_		:      Dict							=					(TFGPTJForCausalLM,) if is_tf_available() else ()
   A_		:      Dict							=					(
       {
           "feature-extraction": TFGPTJModel,
           "question-answering": TFGPTJForQuestionAnswering,
           "text-classification": TFGPTJForSequenceClassification,
           "text-generation": TFGPTJForCausalLM,
           "zero-shot": TFGPTJForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   A_		:      List[str]							=					False
   A_		:      Any							=					False
   A_		:      Optional[int]							=					False
   A_		:      Union[str, Any]							=					False


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	int,			_lowerCamelCase					:	str    ):

     '''simple docstring'''
     if (
         pipeline_test_casse_name == "QAPipelineTests"
         and tokenizer_name is not None
         and not tokenizer_name.endswith('''Fast'''    )
     ):
       # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
       # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
       # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
       return True

     return False


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModelTester(self    )
     __A		     =					ConfigTester(self,			config_class=_lowerCamelCase,			n_embd=37    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     self.config_tester.run_common_tests()


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_past(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_attention_mask_past(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_past_large_inputs(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_lm_head_model(*_lowerCamelCase    )


   @slow
   @unittest.skipIf(
       not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''    )    ) > 0,			'''skip testing on GPU for now to avoid GPU OOM.''',			)
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel.from_pretrained('''EleutherAI/gpt-j-6B''',			from_pt=_lowerCamelCase    )
     self.assertIsNotNone(_lowerCamelCase    )






   @unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''    )
   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     super().test_resize_token_embeddings()





@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class    snake_case						(				unittest.TestCase		):


   '''simple docstring'''

   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			from_pt=_lowerCamelCase    )
     __A		     =					tf.convert_to_tensor([[4_64, 32_90]],			dtype=tf.intaa    )  # The dog
     # fmt: off
     # The dog is a man's best friend. It is a loyal companion, and it is a friend
     __A		     =					[4_64, 32_90, 3_18, 2_57, 5_82, 3_38, 12_66, 15_45, 13, 6_32, 3_18, 2_57, 91_12, 1_51_85, 11, 2_90, 3_40, 3_18, 2_57, 15_45]
     # fmt: on
     __A		     =					model.generate(_lowerCamelCase,			do_sample=_lowerCamelCase    )
     self.assertListEqual(output_ids[0].numpy().tolist(),			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     __A		     =					AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16'''    )
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16''',			from_pt=_lowerCamelCase    )

     __A		     =					tokenizer('''Today is a nice day and''',			return_tensors='''tf'''    )
     # forces the generation to happen on CPU, to avoid GPU-related quirks
     with tf.device(''':/CPU:0'''    ):
       __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			seed=[42, 0]    )
     __A		     =					tokenizer.decode(output_ids[0],			skip_special_tokens=_lowerCamelCase    )

     __A		     =					'''Today is a nice day and I’m going to go for a walk. I’'''
     self.assertEqual(_lowerCamelCase,			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16''',			from_pt=_lowerCamelCase    )
     __A		     =					AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16'''    )

     __A		     =					'''left'''

     # Define PAD Token = EOS Token = 50256
     __A		     =					tokenizer.eos_token
     __A		     =					model.config.eos_token_id

     # use different length sentences to test batching
     __A		     =					[
         '''Hello, my dog is a little''',
         '''Today, I''',
     ]
     __A		     =					[
         '''Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia''',
         '''Today, I’m going to be talking about a topic that’''',
     ]
     return model, tokenizer, sentences, expected_output_sentences


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     # Confirms that we get the expected results with left-padded beam search
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					tokenizer.batch_decode(_lowerCamelCase,			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     # Confirms that left-padding is working properly
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					tokenizer(sentences[0],			return_tensors='''tf'''    )
     __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					(
         shape_list(inputs_non_padded['''input_ids''']    )[-1]
         - tf.reduce_sum(tf.cast(inputs['''attention_mask'''][-1],			tf.intaa    )    ).numpy()
     )
     __A		     =					tokenizer(sentences[1],			return_tensors='''tf'''    )
     __A		     =					model.generate(
         **_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2,			max_length=model.config.max_length - num_paddings    )
     __A		     =					tokenizer.decode(output_non_padded[0],			skip_special_tokens=_lowerCamelCase    )
     __A		     =					tokenizer.decode(output_padded[0],			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			[non_padded_sentence, padded_sentence]    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     # Confirms that XLA is working properly
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					tf.function(model.generate,			jit_compile=_lowerCamelCase    )
     __A		     =					xla_generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					tokenizer.batch_decode(_lowerCamelCase,			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			_lowerCamelCase    )




