






from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.openai.modeling_tf_openai import (
					    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFOpenAIGPTDoubleHeadsModel,
					    TFOpenAIGPTForSequenceClassification,
					    TFOpenAIGPTLMHeadModel,
					    TFOpenAIGPTModel,
					)





class        a__      :






		def __init__(      self							, lowercase__							, )				->				Optional[Any]:
				__A    =							parent
				__A    =							13
				__A    =							7
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							True
				__A    =							99
				__A    =							32
				__A    =							2
				__A    =							4
				__A    =							37
				__A    =							"gelu"
				__A    =							0.1
				__A    =							0.1
				__A    =							512
				__A    =							16
				__A    =							2
				__A    =							0.02
				__A    =							3
				__A    =							4
				__A    =							None
				__A    =							self.vocab_size - 1






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size						)

				__A    =							None
				if self.use_input_mask:
						__A    =							random_attention_mask([self.batch_size, self.seq_length]						)

				__A    =							None
				if self.use_token_type_ids:
						__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size						)

				__A    =							None
				if self.use_mc_token_ids:
						__A    =							ids_tensor([self.batch_size, self.num_choices]							, self.seq_length						)

				__A    =							None
				__A    =							None
				__A    =							None
				if self.use_labels:
						__A    =							ids_tensor([self.batch_size]							, self.type_sequence_label_size						)
						__A    =							ids_tensor([self.batch_size, self.seq_length]							, self.num_labels						)
						__A    =							ids_tensor([self.batch_size]							, self.num_choices						)

				__A    =							OpenAIGPTConfig(
				    vocab_size=self.vocab_size							, n_embd=self.hidden_size							, n_layer=self.num_hidden_layers							, n_head=self.num_attention_heads							, n_positions=self.max_position_embeddings							, pad_token_id=self.pad_token_id							, )

				__A    =							ids_tensor([self.num_hidden_layers, self.num_attention_heads]							, 2						)

				return (
				    config,
				    input_ids,
				    input_mask,
				    head_mask,
				    token_type_ids,
				    mc_token_ids,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Dict:
				__A    =							TFOpenAIGPTModel(config=lowercase__						)
				__A    =							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				__A    =							model(lowercase__						)

				__A    =							[input_ids, input_mask]
				__A    =							model(lowercase__						)

				__A    =							model(lowercase__						)

				self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.hidden_size)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Dict:
				__A    =							TFOpenAIGPTLMHeadModel(config=lowercase__						)
				__A    =							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				__A    =							model(lowercase__						)
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Optional[int]:
				__A    =							TFOpenAIGPTDoubleHeadsModel(config=lowercase__						)

				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)
				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)
				__A    =							tf.tile(tf.expand_dims(lowercase__							, 1						)							, (1, self.num_choices, 1)						)

				__A    =							{
				    "input_ids": multiple_choice_inputs_ids,
				    "mc_token_ids": mc_token_ids,
				    "attention_mask": multiple_choice_input_mask,
				    "token_type_ids": multiple_choice_token_type_ids,
				}
				__A    =							model(lowercase__						)
				self.parent.assertEqual(
				    result.logits.shape							, (self.batch_size, self.num_choices, self.seq_length, self.vocab_size)						)
				self.parent.assertEqual(result.mc_logits.shape							, (self.batch_size, self.num_choices)						)






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, *lowercase__						)				->				Union[str, Any]:
				__A    =							self.num_labels
				__A    =							ids_tensor([self.batch_size]							, self.type_sequence_label_size						)
				__A    =							{
				    "input_ids": input_ids,
				    "attention_mask": input_mask,
				    "token_type_ids": token_type_ids,
				    "labels": sequence_labels,
				}
				__A    =							TFOpenAIGPTForSequenceClassification(lowercase__						)
				__A    =							model(lowercase__						)
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.num_labels)						)






		def 					_lowerCamelCase					(      self						)				->				str:
				__A    =							self.prepare_config_and_inputs()

				(
				    (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   (
				    __A
				) ,   
				)    =							config_and_inputs

				__A    =							{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
				return config, inputs_dict





@require_tf
class        a__      (	lowerCAmelCase__	,       lowerCAmelCase__	,       unittest.TestCase		):
		UpperCAmelCase_							:       Union[str, Any] 					= (
		    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
		    if is_tf_available()
		    else ()
		)
		UpperCAmelCase_							:       Optional[int] 					= (
		    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
		)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
		UpperCAmelCase_							:       List[Any] 					= (
		    {
		        'feature-extraction': TFOpenAIGPTModel,
		        'text-classification': TFOpenAIGPTForSequenceClassification,
		        'text-generation': TFOpenAIGPTLMHeadModel,
		        'zero-shot': TFOpenAIGPTForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		UpperCAmelCase_							:       List[Any] 					= False
		UpperCAmelCase_							:       List[Any] 					= False






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__							, lowercase__							, lowercase__						)				->				Dict:
				if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
						# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
						# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
						# tiny config could not be created.
						return True

				return False






		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							TFOpenAIGPTModelTester(self						)
				__A    =							ConfigTester(self							, config_class=lowercase__							, n_embd=37						)






		def 					_lowerCamelCase					(      self						)				->				List[str]:
				self.config_tester.run_common_tests()






		def 					_lowerCamelCase					(      self						)				->				str:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_model(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Optional[Any]:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_lm_head(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_double_head(*lowercase__						)






		def 					_lowerCamelCase					(      self						)				->				Tuple:
				__A    =							self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase__						)






		@slow
		def 					_lowerCamelCase					(      self						)				->				Optional[int]:
				for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__A    =							TFOpenAIGPTModel.from_pretrained(lowercase__						)
						self.assertIsNotNone(lowercase__						)





@require_tf
class        a__      (	unittest.TestCase		):






		@slow
		def 					_lowerCamelCase					(      self						)				->				Dict:
				__A    =							TFOpenAIGPTLMHeadModel.from_pretrained("openai-gpt"						)
				__A    =							tf.convert_to_tensor([[481, 4735, 544]]							, dtype=tf.intaa						)  # the president is
				__A    =							[
				    481,
				    4735,
				    544,
				    246,
				    963,
				    870,
				    762,
				    239,
				    244,
				    40477,
				    244,
				    249,
				    719,
				    881,
				    487,
				    544,
				    240,
				    244,
				    603,
				    481,
				]  # the president is a very good man. " \n " i\'m sure he is, " said the

				__A    =							model.generate(lowercase__							, do_sample=lowercase__						)
				self.assertListEqual(output_ids[0].numpy().tolist()							, lowercase__						)








"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_mbartaa import MBartaaTokenizer
else:
		_snake_case	:						Dict					=					None


_snake_case	:						int					=					logging.get_logger(__name__)

_snake_case	:						Optional[Any]					=					{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

_snake_case	:						str					=					{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case	:						Union[str, Any]					=					{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
_snake_case	:						int					=					['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class    _UpperCAmelCase  (      lowercase_					):
					UpperCamelCase						=				VOCAB_FILES_NAMES
					UpperCamelCase						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCamelCase						=				PRETRAINED_VOCAB_FILES_MAP
					UpperCamelCase						=				['''input_ids''', '''attention_mask''']
					UpperCamelCase						=				MBartaaTokenizer

					UpperCamelCase						=				[]
					UpperCamelCase						=				[]


					def __init__(   self     :Any	,			__UpperCamelCase     :str=None	,			__UpperCamelCase     :Any=None	,			__UpperCamelCase     :int=None	,			__UpperCamelCase     :Union[str, Any]=None	,			__UpperCamelCase     :Optional[Any]="</s>"	,			__UpperCamelCase     :Tuple="</s>"	,			__UpperCamelCase     :Any="<s>"	,			__UpperCamelCase     :Optional[int]="<unk>"	,			__UpperCamelCase     :str="<pad>"	,			__UpperCamelCase     :int="<mask>"	,			**__UpperCamelCase     :List[str]	,			):
												# Mask token behave like a normal word, i.e. include the space before it
												A						=    AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase						) if isinstance(__UpperCamelCase	,			__UpperCamelCase						) else mask_token

												A						=    kwargs.get("additional_special_tokens"	,			[]						)
												kwargs["additional_special_tokens"] += [
												    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
												]

												super().__init__(
												    __UpperCamelCase	,			src_lang=__UpperCamelCase	,			tgt_lang=__UpperCamelCase	,			tokenizer_file=__UpperCamelCase	,			eos_token=__UpperCamelCase	,			sep_token=__UpperCamelCase	,			cls_token=__UpperCamelCase	,			unk_token=__UpperCamelCase	,			pad_token=__UpperCamelCase	,			mask_token=__UpperCamelCase	,			**__UpperCamelCase	,			)

												A						=    vocab_file
												A						=    False if not self.vocab_file else True

												A						=    {
												    lang_code: self.convert_tokens_to_ids(__UpperCamelCase						) for lang_code in FAIRSEQ_LANGUAGE_CODES
												}

												A						=    src_lang if src_lang is not None else "en_XX"
												A						=    tgt_lang
												A						=    self.lang_code_to_id[self._src_lang]
												self.set_src_lang_special_tokens(self._src_lang						)


					@property
					def       lowerCamelCase			(   self     :int						):
												return self._src_lang


					@src_lang.setter
					def       lowerCamelCase			(   self     :Optional[int]	,			__UpperCamelCase     :str						):
												A						=    new_src_lang
												self.set_src_lang_special_tokens(self._src_lang						)


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :List[int]	,			__UpperCamelCase     :Optional[List[int]] = None						):
												if token_ids_a is None:
																			return self.prefix_tokens + token_ids_a + self.suffix_tokens
												# We don't expect to process pairs, but leave the pair logic for API consistency
												return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :List[str]	,			__UpperCamelCase     :str = "en_XX"	,			__UpperCamelCase     :Optional[List[str]] = None	,			__UpperCamelCase     :str = "ro_RO"	,			**__UpperCamelCase     :Tuple	,			):
												A						=    src_lang
												A						=    tgt_lang
												return super().prepare_seqaseq_batch(__UpperCamelCase	,			__UpperCamelCase	,			**__UpperCamelCase						)


					def       lowerCamelCase			(   self     :str						):
												return self.set_src_lang_special_tokens(self.src_lang						)


					def       lowerCamelCase			(   self     :Optional[Any]						):
												return self.set_tgt_lang_special_tokens(self.tgt_lang						)


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :str						):
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    [self.cur_lang_code_id]
												A						=    [self.eos_token_id]

												A						=    self.convert_ids_to_tokens(self.prefix_tokens						)
												A						=    self.convert_ids_to_tokens(self.suffix_tokens						)

												A						=    processors.TemplateProcessing(
												    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,			self.prefix_tokens + self.suffix_tokens						)						)	,			)


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :str						):
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    [self.cur_lang_code_id]
												A						=    [self.eos_token_id]

												A						=    self.convert_ids_to_tokens(self.prefix_tokens						)
												A						=    self.convert_ids_to_tokens(self.suffix_tokens						)

												A						=    processors.TemplateProcessing(
												    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,			self.prefix_tokens + self.suffix_tokens						)						)	,			)


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :Any	,			__UpperCamelCase     :str	,			__UpperCamelCase     :Optional[str]	,			__UpperCamelCase     :Optional[str]	,			**__UpperCamelCase     :Union[str, Any]						):
												if src_lang is None or tgt_lang is None:
																			raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"						)
												A						=    src_lang
												A						=    self(__UpperCamelCase	,			add_special_tokens=__UpperCamelCase	,			return_tensors=__UpperCamelCase	,			**__UpperCamelCase						)
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    tgt_lang_id
												return inputs


					def       lowerCamelCase			(   self     :Tuple	,			__UpperCamelCase     :str	,			__UpperCamelCase     :Optional[str] = None						):
												if not self.can_save_slow_tokenizer:
																			raise ValueError(
																			    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																			    "tokenizer."						)

												if not os.path.isdir(__UpperCamelCase						):
																			logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
																			return
												A						=    os.path.join(
												    __UpperCamelCase	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

												if os.path.abspath(self.vocab_file						) != os.path.abspath(__UpperCamelCase						):
																			copyfile(self.vocab_file	,			__UpperCamelCase						)

												return (out_vocab_file,)


