



from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers.models.openai.modeling_tf_openai import (
          TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFOpenAIGPTDoubleHeadsModel,
          TFOpenAIGPTForSequenceClassification,
          TFOpenAIGPTLMHeadModel,
          TFOpenAIGPTModel,
      )







class       _snake_case				:






   '''simple docstring'''






   def __init__(    self:     Union[str, Any]  ,lowerCamelCase_:     Union[str, Any]  ,)	->      Optional[Any]:
       UpperCAmelCase_     :       List[str]  		=					parent
       UpperCAmelCase_     :       Optional[int]  		=					13
       UpperCAmelCase_     :       Union[str, Any]  		=					7
       UpperCAmelCase_     :       Tuple  		=					True
       UpperCAmelCase_     :       Union[str, Any]  		=					True
       UpperCAmelCase_     :       List[str]  		=					True
       UpperCAmelCase_     :       Optional[Any]  		=					True
       UpperCAmelCase_     :       str  		=					True
       UpperCAmelCase_     :       Union[str, Any]  		=					99
       UpperCAmelCase_     :       List[str]  		=					32
       UpperCAmelCase_     :       int  		=					2
       UpperCAmelCase_     :       Optional[int]  		=					4
       UpperCAmelCase_     :       Dict  		=					37
       UpperCAmelCase_     :       Any  		=					"""gelu"""
       UpperCAmelCase_     :       str  		=					0.1
       UpperCAmelCase_     :       int  		=					0.1
       UpperCAmelCase_     :       List[str]  		=					512
       UpperCAmelCase_     :       List[str]  		=					16
       UpperCAmelCase_     :       List[str]  		=					2
       UpperCAmelCase_     :       Union[str, Any]  		=					0.0_2
       UpperCAmelCase_     :       Union[str, Any]  		=					3
       UpperCAmelCase_     :       Optional[Any]  		=					4
       UpperCAmelCase_     :       Tuple  		=					None
       UpperCAmelCase_     :       Union[str, Any]  		=					self.vocab_size - 1


   def       A__    (    self:     Tuple							)	->      Dict:
       UpperCAmelCase_     :       Optional[int]  		=					ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size							)

       UpperCAmelCase_     :       str  		=					None
       if self.use_input_mask:
           UpperCAmelCase_     :       str  		=					random_attention_mask([self.batch_size, self.seq_length]							)

       UpperCAmelCase_     :       Any  		=					None
       if self.use_token_type_ids:
           UpperCAmelCase_     :       List[Any]  		=					ids_tensor([self.batch_size, self.seq_length]  ,self.type_vocab_size							)

       UpperCAmelCase_     :       List[str]  		=					None
       if self.use_mc_token_ids:
           UpperCAmelCase_     :       Tuple  		=					ids_tensor([self.batch_size, self.num_choices]  ,self.seq_length							)

       UpperCAmelCase_     :       Tuple  		=					None
       UpperCAmelCase_     :       str  		=					None
       UpperCAmelCase_     :       int  		=					None
       if self.use_labels:
           UpperCAmelCase_     :       List[str]  		=					ids_tensor([self.batch_size]  ,self.type_sequence_label_size							)
           UpperCAmelCase_     :       Dict  		=					ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels							)
           UpperCAmelCase_     :       Union[str, Any]  		=					ids_tensor([self.batch_size]  ,self.num_choices							)

       UpperCAmelCase_     :       Union[str, Any]  		=					OpenAIGPTConfig(
           vocab_size=self.vocab_size  ,n_embd=self.hidden_size  ,n_layer=self.num_hidden_layers  ,n_head=self.num_attention_heads  ,n_positions=self.max_position_embeddings  ,pad_token_id=self.pad_token_id  ,)

       UpperCAmelCase_     :       List[Any]  		=					ids_tensor([self.num_hidden_layers, self.num_attention_heads]  ,2							)

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )


   def       A__    (    self:     str  ,lowerCamelCase_:     int  ,lowerCamelCase_:     str  ,lowerCamelCase_:     Any  ,lowerCamelCase_:     Optional[int]  ,lowerCamelCase_:     Union[str, Any]  ,*lowerCamelCase_:     Dict							)	->      int:
       UpperCAmelCase_     :       Dict  		=					TFOpenAIGPTModel(config=lowerCamelCase_							)
       UpperCAmelCase_     :       Optional[int]  		=					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       UpperCAmelCase_     :       Optional[Any]  		=					model(lowerCamelCase_							)

       UpperCAmelCase_     :       Any  		=					[input_ids, input_mask]
       UpperCAmelCase_     :       int  		=					model(lowerCamelCase_							)

       UpperCAmelCase_     :       List[str]  		=					model(lowerCamelCase_							)

       self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)							)


   def       A__    (    self:     Any  ,lowerCamelCase_:     Optional[Any]  ,lowerCamelCase_:     int  ,lowerCamelCase_:     str  ,lowerCamelCase_:     Union[str, Any]  ,lowerCamelCase_:     str  ,*lowerCamelCase_:     Dict							)	->      List[Any]:
       UpperCAmelCase_     :       Optional[Any]  		=					TFOpenAIGPTLMHeadModel(config=lowerCamelCase_							)
       UpperCAmelCase_     :       Optional[Any]  		=					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       UpperCAmelCase_     :       Union[str, Any]  		=					model(lowerCamelCase_							)
       self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)							)


   def       A__    (    self:     str  ,lowerCamelCase_:     Any  ,lowerCamelCase_:     Optional[Any]  ,lowerCamelCase_:     Tuple  ,lowerCamelCase_:     Optional[Any]  ,lowerCamelCase_:     List[str]  ,lowerCamelCase_:     int  ,*lowerCamelCase_:     str							)	->      int:
       UpperCAmelCase_     :       List[Any]  		=					TFOpenAIGPTDoubleHeadsModel(config=lowerCamelCase_							)

       UpperCAmelCase_     :       str  		=					tf.tile(tf.expand_dims(lowerCamelCase_  ,1							)  ,(1, self.num_choices, 1)							)
       UpperCAmelCase_     :       int  		=					tf.tile(tf.expand_dims(lowerCamelCase_  ,1							)  ,(1, self.num_choices, 1)							)
       UpperCAmelCase_     :       Tuple  		=					tf.tile(tf.expand_dims(lowerCamelCase_  ,1							)  ,(1, self.num_choices, 1)							)

       UpperCAmelCase_     :       Any  		=					{
           """input_ids""": multiple_choice_inputs_ids,
           """mc_token_ids""": mc_token_ids,
           """attention_mask""": multiple_choice_input_mask,
           """token_type_ids""": multiple_choice_token_type_ids,
       }
       UpperCAmelCase_     :       int  		=					model(lowerCamelCase_							)
       self.parent.assertEqual(
           result.logits.shape  ,(self.batch_size, self.num_choices, self.seq_length, self.vocab_size)							)
       self.parent.assertEqual(result.mc_logits.shape  ,(self.batch_size, self.num_choices)							)


   def       A__    (    self:     List[Any]  ,lowerCamelCase_:     Tuple  ,lowerCamelCase_:     int  ,lowerCamelCase_:     Tuple  ,lowerCamelCase_:     int  ,lowerCamelCase_:     Union[str, Any]  ,*lowerCamelCase_:     Dict							)	->      Union[str, Any]:
       UpperCAmelCase_     :       Union[str, Any]  		=					self.num_labels
       UpperCAmelCase_     :       str  		=					ids_tensor([self.batch_size]  ,self.type_sequence_label_size							)
       UpperCAmelCase_     :       List[Any]  		=					{
           """input_ids""": input_ids,
           """attention_mask""": input_mask,
           """token_type_ids""": token_type_ids,
           """labels""": sequence_labels,
       }
       UpperCAmelCase_     :       int  		=					TFOpenAIGPTForSequenceClassification(lowerCamelCase_							)
       UpperCAmelCase_     :       str  		=					model(lowerCamelCase_							)
       self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_labels)							)






   def       A__    (    self:     Dict							)	->      Optional[Any]:
       UpperCAmelCase_     :       Dict  		=					self.prepare_config_and_inputs()

       (
           (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, (
           UpperCAmelCase_
       )		, 
       )     :       List[Any]  		=					config_and_inputs

       UpperCAmelCase_     :       Union[str, Any]  		=					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
       return config, inputs_dict







@require_tf
class       _snake_case				(     __snake_case ,							__snake_case ,							unittest.TestCase ):






   '''simple docstring'''






   A__      : List[Any]       =						(
       (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
       if is_tf_available()
       else ()
   )
   A__      : List[Any]       =						(
       (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
   )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
   A__      : int       =						(
       {
           "feature-extraction": TFOpenAIGPTModel,
           "text-classification": TFOpenAIGPTForSequenceClassification,
           "text-generation": TFOpenAIGPTLMHeadModel,
           "zero-shot": TFOpenAIGPTForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   A__      : List[Any]       =						False
   A__      : Union[str, Any]       =						False


   def       A__    (    self:     str  ,lowerCamelCase_:     Optional[int]  ,lowerCamelCase_:     Union[str, Any]  ,lowerCamelCase_:     Union[str, Any]  ,lowerCamelCase_:     Union[str, Any]  ,lowerCamelCase_:     Tuple							)	->      Tuple:
       if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
           # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
           # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
           # tiny config could not be created.
           return True

       return False


   def       A__    (    self:     Dict							)	->      str:
       UpperCAmelCase_     :       Any  		=					TFOpenAIGPTModelTester(self							)
       UpperCAmelCase_     :       List[Any]  		=					ConfigTester(self  ,config_class=lowerCamelCase_  ,n_embd=37							)


   def       A__    (    self:     List[Any]							)	->      Union[str, Any]:
       self.config_tester.run_common_tests()


   def       A__    (    self:     int							)	->      Union[str, Any]:
       UpperCAmelCase_     :       Union[str, Any]  		=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_model(*lowerCamelCase_							)


   def       A__    (    self:     int							)	->      Tuple:
       UpperCAmelCase_     :       Any  		=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_lm_head(*lowerCamelCase_							)


   def       A__    (    self:     Union[str, Any]							)	->      int:
       UpperCAmelCase_     :       Dict  		=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_double_head(*lowerCamelCase_							)


   def       A__    (    self:     List[str]							)	->      Optional[Any]:
       UpperCAmelCase_     :       Optional[int]  		=					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowerCamelCase_							)






   @slow
   def       A__    (    self:     List[str]							)	->      str:
       for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           UpperCAmelCase_     :       str  		=					TFOpenAIGPTModel.from_pretrained(lowerCamelCase_							)
           self.assertIsNotNone(lowerCamelCase_							)







@require_tf
class       _snake_case				(     unittest.TestCase ):






   '''simple docstring'''






   @slow
   def       A__    (    self:     Dict							)	->      List[str]:
       UpperCAmelCase_     :       int  		=					TFOpenAIGPTLMHeadModel.from_pretrained("""openai-gpt"""							)
       UpperCAmelCase_     :       List[str]  		=					tf.convert_to_tensor([[481, 4735, 544]]  ,dtype=tf.intaa							)  # the president is
       UpperCAmelCase_     :       Union[str, Any]  		=					[
           481,
           4735,
           544,
           246,
           963,
           870,
           762,
           239,
           244,
           40477,
           244,
           249,
           719,
           881,
           487,
           544,
           240,
           244,
           603,
           481,
       ]  # the president is a very good man. " \n " i\'m sure he is, " said the

       UpperCAmelCase_     :       int  		=					model.generate(lowerCamelCase_  ,do_sample=lowerCamelCase_							)
       self.assertListEqual(output_ids[0].numpy().tolist()  ,lowerCamelCase_							)







"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_mbartaa import MBartaaTokenizer
else:
		_snake_case	:						Dict					=					None


_snake_case	:						int					=					logging.get_logger(__name__)

_snake_case	:						Optional[Any]					=					{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

_snake_case	:						str					=					{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case	:						Union[str, Any]					=					{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
_snake_case	:						int					=					['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class    _UpperCAmelCase  (      lowercase_					):
					UpperCamelCase						=				VOCAB_FILES_NAMES
					UpperCamelCase						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCamelCase						=				PRETRAINED_VOCAB_FILES_MAP
					UpperCamelCase						=				['''input_ids''', '''attention_mask''']
					UpperCamelCase						=				MBartaaTokenizer

					UpperCamelCase						=				[]
					UpperCamelCase						=				[]


					def __init__(   self     :Any	,			__UpperCamelCase     :str=None	,			__UpperCamelCase     :Any=None	,			__UpperCamelCase     :int=None	,			__UpperCamelCase     :Union[str, Any]=None	,			__UpperCamelCase     :Optional[Any]="</s>"	,			__UpperCamelCase     :Tuple="</s>"	,			__UpperCamelCase     :Any="<s>"	,			__UpperCamelCase     :Optional[int]="<unk>"	,			__UpperCamelCase     :str="<pad>"	,			__UpperCamelCase     :int="<mask>"	,			**__UpperCamelCase     :List[str]	,			):
												# Mask token behave like a normal word, i.e. include the space before it
												A						=    AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase						) if isinstance(__UpperCamelCase	,			__UpperCamelCase						) else mask_token

												A						=    kwargs.get("additional_special_tokens"	,			[]						)
												kwargs["additional_special_tokens"] += [
												    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
												]

												super().__init__(
												    __UpperCamelCase	,			src_lang=__UpperCamelCase	,			tgt_lang=__UpperCamelCase	,			tokenizer_file=__UpperCamelCase	,			eos_token=__UpperCamelCase	,			sep_token=__UpperCamelCase	,			cls_token=__UpperCamelCase	,			unk_token=__UpperCamelCase	,			pad_token=__UpperCamelCase	,			mask_token=__UpperCamelCase	,			**__UpperCamelCase	,			)

												A						=    vocab_file
												A						=    False if not self.vocab_file else True

												A						=    {
												    lang_code: self.convert_tokens_to_ids(__UpperCamelCase						) for lang_code in FAIRSEQ_LANGUAGE_CODES
												}

												A						=    src_lang if src_lang is not None else "en_XX"
												A						=    tgt_lang
												A						=    self.lang_code_to_id[self._src_lang]
												self.set_src_lang_special_tokens(self._src_lang						)


					@property
					def       lowerCamelCase			(   self     :int						):
												return self._src_lang


					@src_lang.setter
					def       lowerCamelCase			(   self     :Optional[int]	,			__UpperCamelCase     :str						):
												A						=    new_src_lang
												self.set_src_lang_special_tokens(self._src_lang						)


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :List[int]	,			__UpperCamelCase     :Optional[List[int]] = None						):
												if token_ids_a is None:
																			return self.prefix_tokens + token_ids_a + self.suffix_tokens
												# We don't expect to process pairs, but leave the pair logic for API consistency
												return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :List[str]	,			__UpperCamelCase     :str = "en_XX"	,			__UpperCamelCase     :Optional[List[str]] = None	,			__UpperCamelCase     :str = "ro_RO"	,			**__UpperCamelCase     :Tuple	,			):
												A						=    src_lang
												A						=    tgt_lang
												return super().prepare_seqaseq_batch(__UpperCamelCase	,			__UpperCamelCase	,			**__UpperCamelCase						)


					def       lowerCamelCase			(   self     :str						):
												return self.set_src_lang_special_tokens(self.src_lang						)


					def       lowerCamelCase			(   self     :Optional[Any]						):
												return self.set_tgt_lang_special_tokens(self.tgt_lang						)


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :str						):
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    [self.cur_lang_code_id]
												A						=    [self.eos_token_id]

												A						=    self.convert_ids_to_tokens(self.prefix_tokens						)
												A						=    self.convert_ids_to_tokens(self.suffix_tokens						)

												A						=    processors.TemplateProcessing(
												    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,			self.prefix_tokens + self.suffix_tokens						)						)	,			)


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :str						):
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    [self.cur_lang_code_id]
												A						=    [self.eos_token_id]

												A						=    self.convert_ids_to_tokens(self.prefix_tokens						)
												A						=    self.convert_ids_to_tokens(self.suffix_tokens						)

												A						=    processors.TemplateProcessing(
												    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,			self.prefix_tokens + self.suffix_tokens						)						)	,			)


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :Any	,			__UpperCamelCase     :str	,			__UpperCamelCase     :Optional[str]	,			__UpperCamelCase     :Optional[str]	,			**__UpperCamelCase     :Union[str, Any]						):
												if src_lang is None or tgt_lang is None:
																			raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"						)
												A						=    src_lang
												A						=    self(__UpperCamelCase	,			add_special_tokens=__UpperCamelCase	,			return_tensors=__UpperCamelCase	,			**__UpperCamelCase						)
												A						=    self.convert_tokens_to_ids(__UpperCamelCase						)
												A						=    tgt_lang_id
												return inputs


					def       lowerCamelCase			(   self     :Tuple	,			__UpperCamelCase     :str	,			__UpperCamelCase     :Optional[str] = None						):
												if not self.can_save_slow_tokenizer:
																			raise ValueError(
																			    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																			    "tokenizer."						)

												if not os.path.isdir(__UpperCamelCase						):
																			logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
																			return
												A						=    os.path.join(
												    __UpperCamelCase	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

												if os.path.abspath(self.vocab_file						) != os.path.abspath(__UpperCamelCase						):
																			copyfile(self.vocab_file	,			__UpperCamelCase						)

												return (out_vocab_file,)


