

'''simple docstring'''
from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel


def        _lowerCAmelCase					(						_UpperCamelCase						:     Union[str, Any]							, _UpperCamelCase						:     Optional[Any]							, _UpperCamelCase						:     Optional[Any]=None							, _UpperCamelCase						:     List[str]=None				)      ->			int:
     """simple docstring"""




     if attention_mask is None:
          _SCREAMING_SNAKE_CASE							    =tf.cast(tf.math.not_equal(_UpperCamelCase							, config.pad_token_id				)							, tf.inta				)
     return {"input_ids": input_ids, "attention_mask": attention_mask}







@require_tf
class 							A__    :
  A__		 =				OPTConfig
  A__		 =				{}
  A__		 =				'gelu'





  def __init__(		self      :	Optional[Any]							,					_a      :	List[Any]							,					_a      :	List[Any]=13							,					_a      :	Union[str, Any]=7							,					_a      :	Union[str, Any]=True							,					_a      :	Tuple=False							,					_a      :	str=99							,					_a      :	Optional[int]=16							,					_a      :	Union[str, Any]=2							,					_a      :	Dict=4							,					_a      :	Tuple=4							,					_a      :	List[str]="gelu"							,					_a      :	Dict=0.1							,					_a      :	Tuple=0.1							,					_a      :	Dict=20							,					_a      :	Tuple=2							,					_a      :	Union[str, Any]=1							,					_a      :	Any=0							,					_a      :	List[Any]=16							,					_a      :	Optional[Any]=16							,					)	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =parent
       _SCREAMING_SNAKE_CASE							    =batch_size
       _SCREAMING_SNAKE_CASE							    =seq_length
       _SCREAMING_SNAKE_CASE							    =is_training
       _SCREAMING_SNAKE_CASE							    =use_labels
       _SCREAMING_SNAKE_CASE							    =vocab_size
       _SCREAMING_SNAKE_CASE							    =hidden_size
       _SCREAMING_SNAKE_CASE							    =num_hidden_layers
       _SCREAMING_SNAKE_CASE							    =num_attention_heads
       _SCREAMING_SNAKE_CASE							    =intermediate_size
       _SCREAMING_SNAKE_CASE							    =hidden_act
       _SCREAMING_SNAKE_CASE							    =hidden_dropout_prob
       _SCREAMING_SNAKE_CASE							    =attention_probs_dropout_prob
       _SCREAMING_SNAKE_CASE							    =max_position_embeddings
       _SCREAMING_SNAKE_CASE							    =eos_token_id
       _SCREAMING_SNAKE_CASE							    =pad_token_id
       _SCREAMING_SNAKE_CASE							    =bos_token_id
       _SCREAMING_SNAKE_CASE							    =embed_dim
       _SCREAMING_SNAKE_CASE							    =word_embed_proj_dim
       _SCREAMING_SNAKE_CASE							    =False





  def 						A  (		self      :	Optional[Any] )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size, self.seq_length - 1]							,					self.vocab_size )
       _SCREAMING_SNAKE_CASE							    =tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size )							,					1 )
       _SCREAMING_SNAKE_CASE							    =tf.concat([input_ids, eos_tensor]							,					axis=1 )

       _SCREAMING_SNAKE_CASE							    =self.config_cls(
           vocab_size=self.vocab_size							,					hidden_size=self.hidden_size							,					num_hidden_layers=self.num_hidden_layers							,					num_attention_heads=self.num_attention_heads							,					ffn_dim=self.intermediate_size							,					dropout=self.hidden_dropout_prob							,					attention_dropout=self.attention_probs_dropout_prob							,					max_position_embeddings=self.max_position_embeddings							,					eos_token_id=self.eos_token_id							,					bos_token_id=self.bos_token_id							,					pad_token_id=self.pad_token_id							,					embed_dim=self.embed_dim							,					word_embed_proj_dim=self.word_embed_proj_dim							,					is_encoder_decoder=_a							,					**self.config_updates							,					)
       _SCREAMING_SNAKE_CASE							    =prepare_opt_inputs_dict(_a							,					_a )
       return config, inputs_dict





  def 						A  (		self      :	Union[str, Any]							,					_a      :	Tuple							,					_a      :	Optional[Any] )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =TFOPTModel(config=_a )
       _SCREAMING_SNAKE_CASE							    =inputs_dict['input_ids']

       _SCREAMING_SNAKE_CASE							    =input_ids[:1, :]
       _SCREAMING_SNAKE_CASE							    =inputs_dict['attention_mask'][:1, :]
       _SCREAMING_SNAKE_CASE							    =1

       # first forward pass
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					use_cache=_a )

       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =outputs.to_tuple()

       # create hypothetical next token and extent to next_input_ids
       _SCREAMING_SNAKE_CASE							    =ids_tensor((self.batch_size, 3)							,					config.vocab_size )
       _SCREAMING_SNAKE_CASE							    =tf.cast(ids_tensor((self.batch_size, 3)							,					2 )							,					tf.inta )

       # append to next input_ids and
       _SCREAMING_SNAKE_CASE							    =tf.concat([input_ids, next_tokens]							,					axis=-1 )
       _SCREAMING_SNAKE_CASE							    =tf.concat([attention_mask, next_attn_mask]							,					axis=-1 )

       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a )[0]
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					past_key_values=_a )[0]

       self.parent.assertEqual(next_tokens.shape[1]							,					output_from_past.shape[1] )

       # select random slice
       _SCREAMING_SNAKE_CASE							    =int(ids_tensor((1,)							,					output_from_past.shape[-1] ) )
       _SCREAMING_SNAKE_CASE							    =output_from_no_past[:, -3:, random_slice_idx]
       _SCREAMING_SNAKE_CASE							    =output_from_past[:, :, random_slice_idx]

       # test that outputs are equal for slice
       tf.debugging.assert_near(_a							,					_a							,					rtol=1e-3 )







@require_tf
class 							A__    (       A__     ,   A__     ,   unittest.TestCase  ):
  A__		 =				(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
  A__		 =				(TFOPTForCausalLM,) if is_tf_available() else ()
  A__		 =				(
      {'feature-extraction': TFOPTModel, 'text-generation': TFOPTForCausalLM} if is_tf_available() else {}
  )
  A__		 =				False
  A__		 =				False
  A__		 =				False
  A__		 =				10





  def 						A  (		self      :	Union[str, Any] )	->    Optional[int]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =TFOPTModelTester(self )
       _SCREAMING_SNAKE_CASE							    =ConfigTester(self							,					config_class=_a )





  def 						A  (		self      :	str )	->    Optional[int]:





       '''simple docstring'''






       self.config_tester.run_common_tests()





  def 						A  (		self      :	Optional[Any] )	->    str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs_for_common()
       self.model_tester.check_decoder_model_past_large_inputs(*_a )





  def 						A  (		self      :	int )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs_for_common()

       def _get_word_embedding_weight(_a      :	Any							,					_a      :	str ):
            if hasattr(_a							,					'weight' ):
                 return embedding_layer.weight
            else:
                 # Here we build the word embeddings weights if not exists.
                 # And then we retry to get the attribute once built.
                 model.build()
                 if hasattr(_a							,					'weight' ):
                      return embedding_layer.weight
                 else:
                      return None

       for model_class in self.all_model_classes:
            for size in [config.vocab_size - 10, config.vocab_size + 10]:
                 # build the embeddings
                 _SCREAMING_SNAKE_CASE							    =model_class(config=_a )
                 _SCREAMING_SNAKE_CASE							    =_get_word_embedding_weight(_a							,					model.get_input_embeddings() )
                 _SCREAMING_SNAKE_CASE							    =_get_word_embedding_weight(_a							,					model.get_output_embeddings() )

                 # reshape the embeddings
                 model.resize_token_embeddings(_a )
                 _SCREAMING_SNAKE_CASE							    =_get_word_embedding_weight(_a							,					model.get_input_embeddings() )
                 _SCREAMING_SNAKE_CASE							    =_get_word_embedding_weight(_a							,					model.get_output_embeddings() )

                 # check that the resized embeddings size matches the desired size.
                 _SCREAMING_SNAKE_CASE							    =size if size is not None else config.vocab_size

                 self.assertEqual(new_input_embeddings.shape[0]							,					_a )

                 # check that weights remain the same after resizing
                 _SCREAMING_SNAKE_CASE							    =True
                 for pa, pa in zip(old_input_embeddings.value()							,					new_input_embeddings.value() ):
                      if tf.math.reduce_sum(tf.math.abs(pa - pa ) ) > 0:
                           _SCREAMING_SNAKE_CASE							    =False
                 self.assertTrue(_a )

                 if old_output_embeddings is not None and new_output_embeddings is not None:
                      self.assertEqual(new_output_embeddings.shape[0]							,					_a )

                      _SCREAMING_SNAKE_CASE							    =True
                      for pa, pa in zip(old_output_embeddings.value()							,					new_output_embeddings.value() ):
                           if tf.math.reduce_sum(tf.math.abs(pa - pa ) ) > 0:
                                _SCREAMING_SNAKE_CASE							    =False
                      self.assertTrue(_a )


def        _lowerCAmelCase					(						_UpperCamelCase						:     Optional[Any]				)      ->			int:
     """simple docstring"""




     return tf.constant(_UpperCamelCase							, dtype=tf.intaa				)







@require_tf
class 							A__    (       unittest.TestCase  ):
  A__		 =				99





  def 						A  (		self      :	Union[str, Any] )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =tf.ones((4, 1)							,					dtype=tf.intaa ) * 2
       _SCREAMING_SNAKE_CASE							    =tf.concat([ids_tensor((4, 6)							,					self.vocab_size - 3 ) + 3, eos_column_vector]							,					axis=1 )
       _SCREAMING_SNAKE_CASE							    =input_ids.shape[0]
       _SCREAMING_SNAKE_CASE							    =OPTConfig(
           vocab_size=self.vocab_size							,					hidden_size=24							,					num_hidden_layers=2							,					num_attention_heads=2							,					ffn_dim=32							,					max_position_embeddings=48							,					eos_token_id=2							,					pad_token_id=1							,					bos_token_id=0							,					)
       return config, input_ids, batch_size







@require_sentencepiece
@require_tf
class 							A__    (       unittest.TestCase  ):





  @slow
  def 						A  (		self      :	Union[str, Any] )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =TFOPTModel.from_pretrained('facebook/opt-350m' )
       _SCREAMING_SNAKE_CASE							    =_long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]] )
       _SCREAMING_SNAKE_CASE							    =tf.not_equal(_a							,					model.config.pad_token_id )
       with tf.GradientTape():
            _SCREAMING_SNAKE_CASE							    =model(input_ids=_a							,					attention_mask=_a ).last_hidden_state
       _SCREAMING_SNAKE_CASE							    =(1, 11, 512)
       self.assertEqual(output.shape							,					_a )
       _SCREAMING_SNAKE_CASE							    =tf.constant(
           [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]] )
       self.assertTrue(np.allclose(output[:, :3, :3]							,					_a							,					atol=4e-3 ) )

       _SCREAMING_SNAKE_CASE							    =tf.function(_a							,					jit_compile=_a )
       _SCREAMING_SNAKE_CASE							    =xla_generate(_a							,					_a )[0]
       self.assertTrue(np.allclose(output[:, :3, :3]							,					_a							,					atol=4e-2 ) )







@require_tf
@slow
class 							A__    (       unittest.TestCase  ):





  def 						A  (		self      :	List[Any] )	->    Tuple:





       '''simple docstring'''






       super().setUp()
       _SCREAMING_SNAKE_CASE							    ='facebook/opt-350m'





  def 						A  (		self      :	str )	->    Optional[int]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =TFOPTForCausalLM.from_pretrained(self.path_model )
       _SCREAMING_SNAKE_CASE							    =GPTaTokenizer.from_pretrained(self.path_model )

       _SCREAMING_SNAKE_CASE							    =[
           'Today is a beautiful day and I want to',
           'In the city of',
           'Paris is the capital of France and',
           'Computers and mobile phones have taken',
       ]
       # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					return_tensors='tf'							,					padding=_a							,					add_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =tf.math.reduce_mean(model(inputs.input_ids							,					attention_mask=inputs.attention_mask )[0]							,					axis=-1 )
       _SCREAMING_SNAKE_CASE							    =tf.constant(
           [
               [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
               [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
               [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
               [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
           ] )
       self.assertTrue(np.allclose(_a							,					_a							,					atol=1e-4 ) )

       _SCREAMING_SNAKE_CASE							    =tf.function(_a							,					jit_compile=_a )
       _SCREAMING_SNAKE_CASE							    =tf.math.reduce_mean(xla_generate(inputs.input_ids							,					attention_mask=inputs.attention_mask )[0]							,					axis=-1 )
       self.assertTrue(np.allclose(_a							,					_a							,					atol=1e-4 ) )







@require_tf
@slow
class 							A__    (       unittest.TestCase  ):





  @property
  def 						A  (		self      :	List[str] )	->    List[Any]:





       '''simple docstring'''






       return [
           "Today is a beautiful day and I want",
           "In the city of",
           "Paris is the capital of France and",
           "Computers and mobile phones have taken",
       ]





  def 						A  (		self      :	Tuple )	->    Optional[int]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='facebook/opt-125m'

       _SCREAMING_SNAKE_CASE							    =[
           'Today is a beautiful day and I want to',
           'In the city of New York, the city',
           'Paris is the capital of France and the capital',
           'Computers and mobile phones have taken over the',
       ]

       _SCREAMING_SNAKE_CASE							    =[]
       _SCREAMING_SNAKE_CASE							    =GPTaTokenizer.from_pretrained(_a )
       _SCREAMING_SNAKE_CASE							    =TFOPTForCausalLM.from_pretrained(_a )

       for prompt in self.prompts:
            _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					return_tensors='tf' ).input_ids

            _SCREAMING_SNAKE_CASE							    =model.generate(_a							,					max_length=10 )

            _SCREAMING_SNAKE_CASE							    =tokenizer.batch_decode(_a							,					skip_special_tokens=_a )
            predicted_outputs += generated_string

       self.assertListEqual(_a							,					_a )





  def 						A  (		self      :	Dict )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='facebook/opt-350m'

       _SCREAMING_SNAKE_CASE							    =GPTaTokenizer.from_pretrained(_a )
       _SCREAMING_SNAKE_CASE							    =TFOPTForCausalLM.from_pretrained(_a )

       _SCREAMING_SNAKE_CASE							    ='left'

       # use different length sentences to test batching
       _SCREAMING_SNAKE_CASE							    =[
           'Hello, my dog is a little',
           'Today, I',
       ]

       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					return_tensors='tf'							,					padding=_a )
       _SCREAMING_SNAKE_CASE							    =inputs['input_ids']

       _SCREAMING_SNAKE_CASE							    =model.generate(input_ids=_a							,					attention_mask=inputs['attention_mask'] )

       _SCREAMING_SNAKE_CASE							    =tokenizer(sentences[0]							,					return_tensors='tf' ).input_ids
       _SCREAMING_SNAKE_CASE							    =model.generate(input_ids=_a )

       _SCREAMING_SNAKE_CASE							    =inputs_non_padded.shape[-1] - tf.math.reduce_sum(
           tf.cast(inputs['attention_mask'][-1]							,					tf.intaa ) )
       _SCREAMING_SNAKE_CASE							    =tokenizer(sentences[1]							,					return_tensors='tf' ).input_ids
       _SCREAMING_SNAKE_CASE							    =model.generate(input_ids=_a							,					max_length=model.config.max_length - num_paddings )

       _SCREAMING_SNAKE_CASE							    =tokenizer.batch_decode(_a							,					skip_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(output_non_padded[0]							,					skip_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(output_padded[0]							,					skip_special_tokens=_a )

       _SCREAMING_SNAKE_CASE							    =[
           'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
           'Today, I was in the middle of a conversation with a friend about the',
       ]
       self.assertListEqual(_a							,					_a )
       self.assertListEqual(_a							,					[non_padded_sentence, padded_sentence] )





  def 						A  (		self      :	Any )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='facebook/opt-350m'

       _SCREAMING_SNAKE_CASE							    =[
           'Today is a beautiful day and I want to',
           'In the city of San Francisco, the city',
           'Paris is the capital of France and the capital',
           'Computers and mobile phones have taken over the',
       ]

       _SCREAMING_SNAKE_CASE							    =[]
       _SCREAMING_SNAKE_CASE							    =GPTaTokenizer.from_pretrained(_a )
       _SCREAMING_SNAKE_CASE							    =TFOPTForCausalLM.from_pretrained(_a )

       for prompt in self.prompts:
            _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					return_tensors='tf' ).input_ids

            _SCREAMING_SNAKE_CASE							    =model.generate(_a							,					max_length=10 )

            _SCREAMING_SNAKE_CASE							    =tokenizer.batch_decode(_a							,					skip_special_tokens=_a )
            predicted_outputs += generated_string

       self.assertListEqual(_a							,					_a )



'''simple docstring'''
import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp


def        _lowerCAmelCase					(						)      ->			Any:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =ArgumentParser(
         description=(
             'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
         )				)

     # Optional arguments for the launch helper
     parser.add_argument('--num_cores'							, type=_UpperCamelCase							, default=1							, help='Number of TPU cores to use (1 or 8).'				)

     # positional
     parser.add_argument(
         'training_script'							, type=_UpperCamelCase							, help=(
             'The full path to the single TPU training '
             'program/script to be launched in parallel, '
             'followed by all the arguments for the '
             'training script'
         )							, )

     # rest from the training program
     parser.add_argument('training_script_args'							, nargs=_UpperCamelCase				)

     return parser.parse_args()


def        _lowerCAmelCase					(						)      ->			Optional[int]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							    =parse_args()

     # Import training_script as a module.
     _SCREAMING_SNAKE_CASE							    =Path(args.training_script				)
     sys.path.append(str(script_fpath.parent.resolve()				)				)
     _SCREAMING_SNAKE_CASE							    =script_fpath.stem
     _SCREAMING_SNAKE_CASE							    =importlib.import_module(_UpperCamelCase				)

     # Patch sys.argv
     _SCREAMING_SNAKE_CASE							    =[args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores				)]

     xmp.spawn(mod._mp_fn							, args=()							, nprocs=args.num_cores				)


if __name__ == "__main__":
     main()
