






"""simple docstring"""





from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel







@require_tf
class 						_a  :





   '''simple docstring'''






   UpperCamelCase__     	=							MarianConfig
   UpperCamelCase__     	=							{}
   UpperCamelCase__     	=							"""gelu"""





   def __init__(       self					, UpperCAmelCase_					, UpperCAmelCase_=13					, UpperCAmelCase_=7					, UpperCAmelCase_=True					, UpperCAmelCase_=False					, UpperCAmelCase_=99					, UpperCAmelCase_=32					, UpperCAmelCase_=2					, UpperCAmelCase_=4					, UpperCAmelCase_=37					, UpperCAmelCase_=0.1					, UpperCAmelCase_=0.1					, UpperCAmelCase_=20					, UpperCAmelCase_=2					, UpperCAmelCase_=1					, UpperCAmelCase_=0					, )			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     parent
          lowercase__:							List[Any]         =     batch_size
          lowercase__:							Tuple         =     seq_length
          lowercase__:							str         =     is_training
          lowercase__:							Any         =     use_labels
          lowercase__:							str         =     vocab_size
          lowercase__:							Dict         =     hidden_size
          lowercase__:							Union[str, Any]         =     num_hidden_layers
          lowercase__:							Any         =     num_attention_heads
          lowercase__:							Optional[Any]         =     intermediate_size

          lowercase__:							Optional[Any]         =     hidden_dropout_prob
          lowercase__:							List[Any]         =     attention_probs_dropout_prob
          lowercase__:							Dict         =     max_position_embeddings
          lowercase__:							Union[str, Any]         =     eos_token_id
          lowercase__:							Optional[Any]         =     pad_token_id
          lowercase__:							List[Any]         =     bos_token_id





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							List[str]         =     ids_tensor([self.batch_size, self.seq_length - 1]					, self.vocab_size)
          lowercase__:							List[str]         =     tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)					, 1)
          lowercase__:							int         =     tf.concat([input_ids, eos_tensor]					, axis=1)

          lowercase__:							int         =     ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size)

          lowercase__:							List[Any]         =     self.config_cls(
              vocab_size=self.vocab_size					, d_model=self.hidden_size					, encoder_layers=self.num_hidden_layers					, decoder_layers=self.num_hidden_layers					, encoder_attention_heads=self.num_attention_heads					, decoder_attention_heads=self.num_attention_heads					, encoder_ffn_dim=self.intermediate_size					, decoder_ffn_dim=self.intermediate_size					, dropout=self.hidden_dropout_prob					, attention_dropout=self.attention_probs_dropout_prob					, max_position_embeddings=self.max_position_embeddings					, eos_token_ids=[2]					, bos_token_id=self.bos_token_id					, pad_token_id=self.pad_token_id					, decoder_start_token_id=self.pad_token_id					, **self.config_updates					, )
          lowercase__:							str         =     prepare_marian_inputs_dict(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_)
          return config, inputs_dict





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							List[Any]         =     TFMarianModel(config=UpperCAmelCase_).get_decoder()
          lowercase__:							List[str]         =     inputs_dict["input_ids"]

          lowercase__:							Dict         =     input_ids[:1, :]
          lowercase__:							int         =     inputs_dict["attention_mask"][:1, :]
          lowercase__:							int         =     inputs_dict["head_mask"]
          lowercase__:							Tuple         =     1

          # first forward pass
          lowercase__:							Optional[Any]         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_					, head_mask=UpperCAmelCase_					, use_cache=UpperCAmelCase_)

          lowercase__    ,      lowercase__:							int         =     outputs.to_tuple()

          # create hypothetical next token and extent to next_input_ids
          lowercase__:							str         =     ids_tensor((self.batch_size, 3)					, config.vocab_size)
          lowercase__:							str         =     tf.cast(ids_tensor((self.batch_size, 3)					, 2)					, tf.inta)

          # append to next input_ids and
          lowercase__:							List[Any]         =     tf.concat([input_ids, next_tokens]					, axis=-1)
          lowercase__:							str         =     tf.concat([attention_mask, next_attn_mask]					, axis=-1)

          lowercase__:							Any         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_)[0]
          lowercase__:							Tuple         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_					, past_key_values=UpperCAmelCase_)[0]

          self.parent.assertEqual(next_tokens.shape[1]					, output_from_past.shape[1])

          # select random slice
          lowercase__:							Optional[Any]         =     int(ids_tensor((1,)					, output_from_past.shape[-1]))
          lowercase__:							List[str]         =     output_from_no_past[:, -3:, random_slice_idx]
          lowercase__:							Optional[int]         =     output_from_past[:, :, random_slice_idx]

          # test that outputs are equal for slice
          tf.debugging.assert_near(UpperCAmelCase_					, UpperCAmelCase_					, rtol=1E-3)







def 							A(    snake_case_		,  snake_case_		,  snake_case_		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  ):
       """simple docstring"""





       if attention_mask is None:
              lowercase__:							Optional[int]         =     tf.cast(tf.math.not_equal(snake_case_		,  config.pad_token_id						)		,  tf.inta						)
       if decoder_attention_mask is None:
              lowercase__:							Optional[Any]         =     tf.concat(
                  [
                      tf.ones(decoder_input_ids[:, :1].shape		,  dtype=tf.inta						),
                      tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]		,  config.pad_token_id						)		,  tf.inta						),
                  ]		,  axis=-1		,  )
       if head_mask is None:
              lowercase__:							str         =     tf.ones((config.encoder_layers, config.encoder_attention_heads)						)
       if decoder_head_mask is None:
              lowercase__:							Union[str, Any]         =     tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
       if cross_attn_head_mask is None:
              lowercase__:							str         =     tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
       return {
           "input_ids": input_ids,
           "decoder_input_ids": decoder_input_ids,
           "attention_mask": attention_mask,
           "decoder_attention_mask": decoder_attention_mask,
           "head_mask": head_mask,
           "decoder_head_mask": decoder_head_mask,
           "cross_attn_head_mask": cross_attn_head_mask,
       }







@require_tf
class 						_a  (		lowercase_							,	lowercase_							,	unittest.TestCase       ):





   '''simple docstring'''






   UpperCamelCase__     	=							(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
   UpperCamelCase__     	=							(TFMarianMTModel,) if is_tf_available() else ()
   UpperCamelCase__     	=							(
       {
           """conversational""": TFMarianMTModel,
           """feature-extraction""": TFMarianModel,
           """summarization""": TFMarianMTModel,
           """text2text-generation""": TFMarianMTModel,
           """translation""": TFMarianMTModel,
       }
       if is_tf_available()
       else {}
   )
   UpperCamelCase__     	=							True
   UpperCamelCase__     	=							False
   UpperCamelCase__     	=							False





   def __lowercase							(       self)			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Tuple         =     TFMarianModelTester(self)
          lowercase__:							Dict         =     ConfigTester(self					, config_class=UpperCAmelCase_)





   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          self.config_tester.run_common_tests()





   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          lowercase__:							List[Any]         =     self.model_tester.prepare_config_and_inputs_for_common()
          self.model_tester.check_decoder_model_past_large_inputs(*UpperCAmelCase_)







@require_tf
class 						_a  (		unittest.TestCase       ):





   '''simple docstring'''






   UpperCamelCase__     	=							1_000  # show more chars for failing integration tests





   @classmethod
   def __lowercase							(       cls)			->   None:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     F"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
          return cls





   @cached_property
   def __lowercase							(       self)			->   MarianTokenizer:




          '''simple docstring'''




          return AutoTokenizer.from_pretrained(self.model_name)





   @property
   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          return self.tokenizer.eos_token_id





   @cached_property
   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          warnings.simplefilter("error")
          lowercase__:							TFMarianMTModel         =     TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name)
          assert isinstance(UpperCAmelCase_					, UpperCAmelCase_)
          lowercase__:							Optional[Any]         =     model.config
          self.assertListEqual(c.bad_words_ids					, [[c.pad_token_id]])
          self.assertEqual(c.max_length					, 512)
          self.assertEqual(c.decoder_start_token_id					, c.pad_token_id)
          return model





   def __lowercase							(       self					, **UpperCAmelCase_)			->   Tuple:




          '''simple docstring'''




          lowercase__:							List[str]         =     self.translate_src_text(**UpperCAmelCase_)
          self.assertListEqual(self.expected_text					, UpperCAmelCase_)





   def __lowercase							(       self					, **UpperCAmelCase_)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							int         =     self.tokenizer(self.src_text					, **UpperCAmelCase_					, padding=UpperCAmelCase_					, return_tensors="tf")
          lowercase__:							Optional[Any]         =     self.model.generate(
              model_inputs.input_ids					, attention_mask=model_inputs.attention_mask					, num_beams=2					, max_length=128)
          lowercase__:							Optional[Any]         =     self.tokenizer.batch_decode(generated_ids.numpy()					, skip_special_tokens=UpperCAmelCase_)
          return generated_words







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""mt"""
   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
   UpperCamelCase__     	=							["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							"""zh"""
   UpperCamelCase__     	=							["""My name is Wolfgang and I live in Berlin"""]
   UpperCamelCase__     	=							["""我叫沃尔夫冈 我住在柏林"""]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   List[str]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							"""ROMANCE"""
   UpperCamelCase__     	=							[
       """>>fr<< Don't spend so much time watching TV.""",
       """>>pt<< Your message has been sent.""",
       """>>es<< He's two years older than me.""",
   ]
   UpperCamelCase__     	=							[
       """Ne passez pas autant de temps à regarder la télé.""",
       """A sua mensagem foi enviada.""",
       """Es dos años más viejo que yo.""",
   ]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Any         =     TranslationPipeline(self.model					, self.tokenizer					, framework="tf")
          lowercase__:							Optional[int]         =     pipeline(self.src_text)
          self.assertEqual(self.expected_text					, [x["translation_text"] for x in output])










import unittest

import pytest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    MBartaaTokenizerFast,
    MBartConfig,
    MBartForConditionalGeneration,
    TranslationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow

from .test_pipelines_common import ANY







@is_pipeline_test
class  lowerCamelCase__  (			unittest.TestCase):
     SCREAMING_SNAKE_CASE__      =     MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
     SCREAMING_SNAKE_CASE__      =     TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING





     def 	__A   (self						,	UpperCAmelCase						,	UpperCAmelCase						,	UpperCAmelCase			)	->  Tuple:
      if isinstance(model.config						,	UpperCAmelCase			):
       _lowercase ,       _lowercase				 =list(tokenizer.lang_code_to_id.keys()			)[:2]
       _lowercase				 =TranslationPipeline(model=UpperCAmelCase						,	tokenizer=UpperCAmelCase						,	src_lang=UpperCAmelCase						,	tgt_lang=UpperCAmelCase			)
      else:
       _lowercase				 =TranslationPipeline(model=UpperCAmelCase						,	tokenizer=UpperCAmelCase			)
      return translator, ["Some string", "Some other text"]





     def 	__A   (self						,	UpperCAmelCase						,	UpperCAmelCase			)	->  List[Any]:
      _lowercase				 =translator('''Some string'''			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': ANY(UpperCAmelCase			)}]			)

      _lowercase				 =translator(['''Some string''']			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': ANY(UpperCAmelCase			)}]			)

      _lowercase				 =translator(['''Some string''', '''other string''']			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': ANY(UpperCAmelCase			)}, {'''translation_text''': ANY(UpperCAmelCase			)}]			)





     @require_torch
     def 	__A   (self			)	->  Optional[Any]:
      _lowercase				 =pipeline('''translation_en_to_ro'''						,	model='''patrickvonplaten/t5-tiny-random'''						,	framework='''pt'''			)
      _lowercase				 =translator('''This is a test string'''						,	max_length=2_0			)
      self.assertEqual(
          UpperCAmelCase						,	[
              {
                  '''translation_text''': (
                      '''Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide'''
                      ''' Beide Beide'''
                  )
              }
          ]						,	)





     @require_tf
     def 	__A   (self			)	->  Optional[Any]:
      _lowercase				 =pipeline('''translation_en_to_ro'''						,	model='''patrickvonplaten/t5-tiny-random'''						,	framework='''tf'''			)
      _lowercase				 =translator('''This is a test string'''						,	max_length=2_0			)
      self.assertEqual(
          UpperCAmelCase						,	[
              {
                  '''translation_text''': (
                      '''Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide'''
                      ''' Beide Beide'''
                  )
              }
          ]						,	)





     @require_torch
     def 	__A   (self			)	->  str:
      _lowercase				 =pipeline('''translation_en_to_de'''						,	model='''patrickvonplaten/t5-tiny-random'''						,	framework='''pt'''			)
      _lowercase				 =translator('''This is a test string'''						,	max_length=2_0			)
      self.assertEqual(
          UpperCAmelCase						,	[
              {
                  '''translation_text''': (
                      '''monoton monoton monoton monoton monoton monoton monoton monoton monoton monoton urine urine'''
                      ''' urine urine urine urine urine urine urine'''
                  )
              }
          ]						,	)





     @require_tf
     def 	__A   (self			)	->  Optional[int]:
      _lowercase				 =pipeline('''translation_en_to_de'''						,	model='''patrickvonplaten/t5-tiny-random'''						,	framework='''tf'''			)
      _lowercase				 =translator('''This is a test string'''						,	max_length=2_0			)
      self.assertEqual(
          UpperCAmelCase						,	[
              {
                  '''translation_text''': (
                      '''monoton monoton monoton monoton monoton monoton monoton monoton monoton monoton urine urine'''
                      ''' urine urine urine urine urine urine urine'''
                  )
              }
          ]						,	)







class  lowerCamelCase__  (			unittest.TestCase):





     @require_torch
     @slow
     def 	__A   (self			)	->  Optional[Any]:
      # We don't provide a default for this pair
      with self.assertRaises(UpperCAmelCase			):
       pipeline(task='''translation_cn_to_ar'''			)

      # but we do for this one
      _lowercase				 =pipeline(task='''translation_en_to_de'''			)
      self.assertEqual(translator._preprocess_params['''src_lang''']						,	'''en'''			)
      self.assertEqual(translator._preprocess_params['''tgt_lang''']						,	'''de'''			)





     @require_torch
     @slow
     def 	__A   (self			)	->  Optional[Any]:
      _lowercase				 =MBartForConditionalGeneration.from_pretrained('''facebook/mbart-large-50-many-to-many-mmt'''			)
      _lowercase				 =MBartaaTokenizerFast.from_pretrained('''facebook/mbart-large-50-many-to-many-mmt'''			)

      _lowercase				 =pipeline(task='''translation'''						,	model=UpperCAmelCase						,	tokenizer=UpperCAmelCase			)
      # Missing src_lang, tgt_lang
      with self.assertRaises(UpperCAmelCase			):
       translator('''This is a test'''			)

      _lowercase				 =translator('''This is a test'''						,	src_lang='''en_XX'''						,	tgt_lang='''ar_AR'''			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': '''هذا إختبار'''}]			)

      _lowercase				 =translator('''This is a test'''						,	src_lang='''en_XX'''						,	tgt_lang='''hi_IN'''			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': '''यह एक परीक्षण है'''}]			)

      # src_lang, tgt_lang can be defined at pipeline call time
      _lowercase				 =pipeline(task='''translation'''						,	model=UpperCAmelCase						,	tokenizer=UpperCAmelCase						,	src_lang='''en_XX'''						,	tgt_lang='''ar_AR'''			)
      _lowercase				 =translator('''This is a test'''			)
      self.assertEqual(UpperCAmelCase						,	[{'''translation_text''': '''هذا إختبار'''}]			)





     @require_torch
     def 	__A   (self			)	->  Tuple:
      _lowercase				 ='''patrickvonplaten/t5-tiny-random'''
      _lowercase				 =pipeline(task='''translation_cn_to_ar'''						,	model=UpperCAmelCase			)
      self.assertEqual(translator._preprocess_params['''src_lang''']						,	'''cn'''			)
      self.assertEqual(translator._preprocess_params['''tgt_lang''']						,	'''ar'''			)





     @require_torch
     def 	__A   (self			)	->  str:
      _lowercase				 ='''patrickvonplaten/t5-tiny-random'''
      with pytest.warns(UpperCAmelCase						,	match=R'''.*translation_en_to_de.*'''			):
       _lowercase				 =pipeline(task='''translation'''						,	model=UpperCAmelCase			)
      self.assertEqual(translator.task						,	'''translation_en_to_de'''			)
      self.assertEqual(translator._preprocess_params['''src_lang''']						,	'''en'''			)
      self.assertEqual(translator._preprocess_params['''tgt_lang''']						,	'''de'''			)





     @require_torch
     def 	__A   (self			)	->  List[str]:
      with self.assertRaises(UpperCAmelCase			):
       pipeline(task='''translation'''			)

