import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


__lowerCAmelCase    :  Dict										= threading.Lock()
__lowerCAmelCase    :  Optional[logging.Handler]										= None

__lowerCAmelCase    :  Union[str, Any]										= {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

__lowerCAmelCase    :  str										= logging.WARNING

__lowerCAmelCase    :  Any										= True

def   a_					()->				Union[str, Any]:
    snake_case:						List[Any]         =   os.getenv("""DIFFUSERS_VERBOSITY"""			,	_lowerCAmelCase       )
    if env_level_str:
        if env_level_str in log_levels:
            return log_levels[env_level_str]
        else:
            logging.getLogger().warning(
                F"Unknown option DIFFUSERS_VERBOSITY={env_level_str}, "
                F"has to be one of: { ', '.join(log_levels.keys()       ) }"       )
    return _default_log_level

def   a_					()->				str:
    return __name__.split("""."""       )[0]

def   a_					()->				logging.Logger:
    return logging.getLogger(_get_library_name()       )

def   a_					()->				None:
    global _default_handler

    with _lock:
        if _default_handler:
            # This library has already configured the library root logger.
            return
        snake_case:						Tuple         =   logging.StreamHandler()  # Set sys.stderr as stream.
        snake_case:						Any         =   sys.stderr.flush

        # Apply our default configuration to the library root logger.
        snake_case:						Optional[int]         =   _get_library_root_logger()
        library_root_logger.addHandler(_default_handler       )
        library_root_logger.setLevel(_get_default_logging_level()       )
        snake_case:						str         =   False

def   a_					()->				None:
    global _default_handler

    with _lock:
        if not _default_handler:
            return

        snake_case:						Union[str, Any]         =   _get_library_root_logger()
        library_root_logger.removeHandler(_default_handler       )
        library_root_logger.setLevel(logging.NOTSET       )
        snake_case:						Optional[Any]         =   None

def   a_					()->				Optional[int]:
    return log_levels

def   a_					(_lowerCAmelCase  :   Optional[str] = None       )->				logging.Logger:

    if name is None:
        snake_case:						List[str]         =   _get_library_name()

    _configure_library_root_logger()
    return logging.getLogger(_lowerCAmelCase       )

def   a_					()->				int:
    _configure_library_root_logger()
    return _get_library_root_logger().getEffectiveLevel()

def   a_					(_lowerCAmelCase  :   int       )->				None:
    _configure_library_root_logger()
    _get_library_root_logger().setLevel(_lowerCAmelCase       )

def   a_					()->				Tuple:
    return set_verbosity(_lowerCAmelCase       )

def   a_					()->				Optional[Any]:
    return set_verbosity(_lowerCAmelCase       )

def   a_					()->				Optional[Any]:
    return set_verbosity(_lowerCAmelCase       )

def   a_					()->				Tuple:
    return set_verbosity(_lowerCAmelCase       )

def   a_					()->				None:
    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().removeHandler(_default_handler       )

def   a_					()->				None:
    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().addHandler(_default_handler       )

def   a_					(_lowerCAmelCase  :   logging.Handler       )->				None:
    _configure_library_root_logger()

    assert handler is not None
    _get_library_root_logger().addHandler(_lowerCAmelCase       )

def   a_					(_lowerCAmelCase  :   logging.Handler       )->				None:
    _configure_library_root_logger()

    assert handler is not None and handler not in _get_library_root_logger().handlers
    _get_library_root_logger().removeHandler(_lowerCAmelCase       )

def   a_					()->				None:
    _configure_library_root_logger()
    snake_case:						Tuple         =   False

def   a_					()->				None:
    _configure_library_root_logger()
    snake_case:						List[Any]         =   True

def   a_					()->				None:
    snake_case:						int         =   _get_library_root_logger().handlers

    for handler in handlers:
        snake_case:						Union[str, Any]         =   logging.Formatter("""[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"""       )
        handler.setFormatter(_lowerCAmelCase       )

def   a_					()->				None:
    snake_case:						str         =   _get_library_root_logger().handlers

    for handler in handlers:
        handler.setFormatter(_lowerCAmelCase       )

def   a_					(self  :   Union[str, Any]			,	*_lowerCAmelCase  :   Optional[Any]			,	**_lowerCAmelCase  :   Tuple       )->				Union[str, Any]:
    snake_case:						int         =   os.getenv("""DIFFUSERS_NO_ADVISORY_WARNINGS"""			,	_lowerCAmelCase       )
    if no_advisory_warnings:
        return
    self.warning(*_lowerCAmelCase			,	**_lowerCAmelCase       )


__lowerCAmelCase    :  str										= warning_advice
class 				lowerCamelCase			:






      def __init__(      self  ,		*__lowerCamelCase  ,		**__lowerCamelCase						)     ->				str:  # pylint: disable=unused-argument


          '''simple docstring'''







          snake_case:						List[Any]         =   args[0] if args else None






      def __iter__(      self						)     ->				Optional[int]:


          '''simple docstring'''







          return iter(self._iterator						)






      def __getattr__(      self  ,		__lowerCamelCase						)     ->				Any:


          '''simple docstring'''







          def empty_fn(*__lowerCamelCase  ,		**__lowerCamelCase						):  # pylint: disable=unused-argument
              return

          return empty_fn






      def __enter__(      self						)     ->				int:


          '''simple docstring'''







          return self






      def __exit__(      self  ,		__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)     ->				Union[str, Any]:


          '''simple docstring'''







          return





class 				lowerCamelCase			:






      def __call__(      self  ,		*__lowerCamelCase  ,		**__lowerCamelCase						)     ->				str:


          '''simple docstring'''







          if _tqdm_active:
              return tqdm_lib.tqdm(*__lowerCamelCase  ,		**__lowerCamelCase						)
          else:
              return EmptyTqdm(*__lowerCamelCase  ,		**__lowerCamelCase						)






      def 			lowerCAmelCase_   (      self  ,		*__lowerCamelCase  ,		**__lowerCamelCase						)     ->				Tuple:


          '''simple docstring'''







          snake_case:						List[str]         =   None
          if _tqdm_active:
              return tqdm_lib.tqdm.set_lock(*__lowerCamelCase  ,		**__lowerCamelCase						)






      def 			lowerCAmelCase_   (      self						)     ->				str:


          '''simple docstring'''







          if _tqdm_active:
              return tqdm_lib.tqdm.get_lock()





__lowerCAmelCase    :  Dict										= _tqdm_cls()

def   a_					()->				bool:
    global _tqdm_active
    return bool(_tqdm_active       )

def   a_					()->				Optional[Any]:
    global _tqdm_active
    snake_case:						Optional[Any]         =   True




def   a_					()->				Tuple:
    global _tqdm_active
    snake_case:						Any         =   False



from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel
@require_tf
class 				lowerCamelCase			:
      __lowerCamelCase							=					MarianConfig
      __lowerCamelCase							=					{}
      __lowerCamelCase							=					'gelu'






      def __init__(      self  ,		__lowerCamelCase  ,		__lowerCamelCase=13  ,		__lowerCamelCase=7  ,		__lowerCamelCase=True  ,		__lowerCamelCase=False  ,		__lowerCamelCase=99  ,		__lowerCamelCase=32  ,		__lowerCamelCase=2  ,		__lowerCamelCase=4  ,		__lowerCamelCase=37  ,		__lowerCamelCase=0.1  ,		__lowerCamelCase=0.1  ,		__lowerCamelCase=20  ,		__lowerCamelCase=2  ,		__lowerCamelCase=1  ,		__lowerCamelCase=0  ,		)     ->				Any:


          '''simple docstring'''







          snake_case:						Any         =   parent
          snake_case:						List[Any]         =   batch_size
          snake_case:						int         =   seq_length
          snake_case:						Tuple         =   is_training
          snake_case:						Union[str, Any]         =   use_labels
          snake_case:						Union[str, Any]         =   vocab_size
          snake_case:						Dict         =   hidden_size
          snake_case:						Any         =   num_hidden_layers
          snake_case:						List[Any]         =   num_attention_heads
          snake_case:						str         =   intermediate_size

          snake_case:						Union[str, Any]         =   hidden_dropout_prob
          snake_case:						List[Any]         =   attention_probs_dropout_prob
          snake_case:						Tuple         =   max_position_embeddings
          snake_case:						Optional[int]         =   eos_token_id
          snake_case:						Optional[Any]         =   pad_token_id
          snake_case:						Optional[Any]         =   bos_token_id






      def 			lowerCAmelCase_   (      self						)     ->				Optional[int]:


          '''simple docstring'''







          snake_case:						Tuple         =   ids_tensor([self.batch_size, self.seq_length - 1]  ,		self.vocab_size						)
          snake_case:						Union[str, Any]         =   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size						)  ,		1						)
          snake_case:						int         =   tf.concat([input_ids, eos_tensor]  ,		axis=1						)

          snake_case:						Optional[int]         =   ids_tensor([self.batch_size, self.seq_length]  ,		self.vocab_size						)

          snake_case:						List[str]         =   self.config_cls(
              vocab_size=self.vocab_size  ,		d_model=self.hidden_size  ,		encoder_layers=self.num_hidden_layers  ,		decoder_layers=self.num_hidden_layers  ,		encoder_attention_heads=self.num_attention_heads  ,		decoder_attention_heads=self.num_attention_heads  ,		encoder_ffn_dim=self.intermediate_size  ,		decoder_ffn_dim=self.intermediate_size  ,		dropout=self.hidden_dropout_prob  ,		attention_dropout=self.attention_probs_dropout_prob  ,		max_position_embeddings=self.max_position_embeddings  ,		eos_token_ids=[2]  ,		bos_token_id=self.bos_token_id  ,		pad_token_id=self.pad_token_id  ,		decoder_start_token_id=self.pad_token_id  ,		**self.config_updates  ,		)
          snake_case:						Tuple         =   prepare_marian_inputs_dict(__lowerCamelCase  ,		__lowerCamelCase  ,		__lowerCamelCase						)
          return config, inputs_dict






      def 			lowerCAmelCase_   (      self  ,		__lowerCamelCase  ,		__lowerCamelCase						)     ->				List[str]:


          '''simple docstring'''







          snake_case:						Any         =   TFMarianModel(config=__lowerCamelCase						).get_decoder()
          snake_case:						str         =   inputs_dict["""input_ids"""]

          snake_case:						Union[str, Any]         =   input_ids[:1, :]
          snake_case:						Union[str, Any]         =   inputs_dict["""attention_mask"""][:1, :]
          snake_case:						str         =   inputs_dict["""head_mask"""]
          snake_case:						Optional[Any]         =   1

          # first forward pass
          snake_case:						int         =   model(__lowerCamelCase  ,		attention_mask=__lowerCamelCase  ,		head_mask=__lowerCamelCase  ,		use_cache=__lowerCamelCase						)

          snake_case    , snake_case:						List[str]         =   outputs.to_tuple()

          # create hypothetical next token and extent to next_input_ids
          snake_case:						Optional[int]         =   ids_tensor((self.batch_size, 3)  ,		config.vocab_size						)
          snake_case:						List[Any]         =   tf.cast(ids_tensor((self.batch_size, 3)  ,		2						)  ,		tf.inta						)

          # append to next input_ids and
          snake_case:						Optional[int]         =   tf.concat([input_ids, next_tokens]  ,		axis=-1						)
          snake_case:						Dict         =   tf.concat([attention_mask, next_attn_mask]  ,		axis=-1						)

          snake_case:						Optional[Any]         =   model(__lowerCamelCase  ,		attention_mask=__lowerCamelCase						)[0]
          snake_case:						Any         =   model(__lowerCamelCase  ,		attention_mask=__lowerCamelCase  ,		past_key_values=__lowerCamelCase						)[0]

          self.parent.assertEqual(next_tokens.shape[1]  ,		output_from_past.shape[1]						)

          # select random slice
          snake_case:						str         =   int(ids_tensor((1,)  ,		output_from_past.shape[-1]						)						)
          snake_case:						List[Any]         =   output_from_no_past[:, -3:, random_slice_idx]
          snake_case:						Optional[int]         =   output_from_past[:, :, random_slice_idx]

          # test that outputs are equal for slice
          tf.debugging.assert_near(__lowerCamelCase  ,		__lowerCamelCase  ,		rtol=1e-3						)





def   a_					(_lowerCAmelCase  :   Union[str, Any]			,	_lowerCAmelCase  :   Tuple			,	_lowerCAmelCase  :   int			,	_lowerCAmelCase  :   Tuple=None			,	_lowerCAmelCase  :   Optional[int]=None			,	_lowerCAmelCase  :   Optional[int]=None			,	_lowerCAmelCase  :   Optional[Any]=None			,	_lowerCAmelCase  :   List[str]=None			,	)->				Any:
    if attention_mask is None:
        snake_case:						str         =   tf.cast(tf.math.not_equal(_lowerCAmelCase			,	config.pad_token_id       )			,	tf.inta       )
    if decoder_attention_mask is None:
        snake_case:						Dict         =   tf.concat(
            [
                tf.ones(decoder_input_ids[:, :1].shape			,	dtype=tf.inta       ),
                tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]			,	config.pad_token_id       )			,	tf.inta       ),
            ]			,	axis=-1			,	)
    if head_mask is None:
        snake_case:						Union[str, Any]         =   tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
    if decoder_head_mask is None:
        snake_case:						Optional[Any]         =   tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
    if cross_attn_head_mask is None:
        snake_case:						Optional[int]         =   tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }
@require_tf
class 				lowerCamelCase			(							__snake_case				,     __snake_case				,     unittest.TestCase					):
      __lowerCamelCase							=					(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
      __lowerCamelCase							=					(TFMarianMTModel,) if is_tf_available() else ()
      __lowerCamelCase							=					(
          {
              'conversational': TFMarianMTModel,
              'feature-extraction': TFMarianModel,
              'summarization': TFMarianMTModel,
              'text2text-generation': TFMarianMTModel,
              'translation': TFMarianMTModel,
          }
          if is_tf_available()
          else {}
      )
      __lowerCamelCase							=					True
      __lowerCamelCase							=					False
      __lowerCamelCase							=					False






      def 			lowerCAmelCase_   (      self						)     ->				Dict:


          '''simple docstring'''







          snake_case:						int         =   TFMarianModelTester(self						)
          snake_case:						Tuple         =   ConfigTester(self  ,		config_class=__lowerCamelCase						)






      def 			lowerCAmelCase_   (      self						)     ->				int:


          '''simple docstring'''







          self.config_tester.run_common_tests()






      def 			lowerCAmelCase_   (      self						)     ->				Any:


          '''simple docstring'''







          snake_case:						int         =   self.model_tester.prepare_config_and_inputs_for_common()
          self.model_tester.check_decoder_model_past_large_inputs(*__lowerCamelCase						)





@require_tf
class 				lowerCamelCase			(							unittest.TestCase					):
      __lowerCamelCase							=					1_000  # show more chars for failing integration tests






      @classmethod
      def 			lowerCAmelCase_   (      cls						)     ->				None:


          '''simple docstring'''







          snake_case:						Any         =   F"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
          return cls






      @cached_property
      def 			lowerCAmelCase_   (      self						)     ->				MarianTokenizer:


          '''simple docstring'''







          return AutoTokenizer.from_pretrained(self.model_name						)






      @property
      def 			lowerCAmelCase_   (      self						)     ->				int:


          '''simple docstring'''







          return self.tokenizer.eos_token_id






      @cached_property
      def 			lowerCAmelCase_   (      self						)     ->				Tuple:


          '''simple docstring'''







          warnings.simplefilter("""error"""						)
          snake_case:						TFMarianMTModel         =   TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name						)
          assert isinstance(__lowerCamelCase  ,		__lowerCamelCase						)
          snake_case:						Optional[Any]         =   model.config
          self.assertListEqual(c.bad_words_ids  ,		[[c.pad_token_id]]						)
          self.assertEqual(c.max_length  ,		5_12						)
          self.assertEqual(c.decoder_start_token_id  ,		c.pad_token_id						)
          return model






      def 			lowerCAmelCase_   (      self  ,		**__lowerCamelCase						)     ->				Optional[int]:


          '''simple docstring'''







          snake_case:						Union[str, Any]         =   self.translate_src_text(**__lowerCamelCase						)
          self.assertListEqual(self.expected_text  ,		__lowerCamelCase						)






      def 			lowerCAmelCase_   (      self  ,		**__lowerCamelCase						)     ->				Dict:


          '''simple docstring'''







          snake_case:						Any         =   self.tokenizer(self.src_text  ,		**__lowerCamelCase  ,		padding=__lowerCamelCase  ,		return_tensors="""tf"""						)
          snake_case:						int         =   self.model.generate(
              model_inputs.input_ids  ,		attention_mask=model_inputs.attention_mask  ,		num_beams=2  ,		max_length=1_28						)
          snake_case:						Any         =   self.tokenizer.batch_decode(generated_ids.numpy()  ,		skip_special_tokens=__lowerCamelCase						)
          return generated_words





@require_sentencepiece
@require_tokenizers
@require_tf
class 				lowerCamelCase			(							__snake_case					):
      __lowerCamelCase							=					'mt'
      __lowerCamelCase							=					'en'
      __lowerCamelCase							=					['Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.']
      __lowerCamelCase							=					['Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.']






      @unittest.skip("""Skipping until #12647 is resolved."""						)
      @slow
      def 			lowerCAmelCase_   (      self						)     ->				Optional[int]:


          '''simple docstring'''







          self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class 				lowerCamelCase			(							__snake_case					):
      __lowerCamelCase							=					'en'
      __lowerCamelCase							=					'zh'
      __lowerCamelCase							=					['My name is Wolfgang and I live in Berlin']
      __lowerCamelCase							=					['我叫沃尔夫冈 我住在柏林']






      @unittest.skip("""Skipping until #12647 is resolved."""						)
      @slow
      def 			lowerCAmelCase_   (      self						)     ->				str:


          '''simple docstring'''







          self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class 				lowerCamelCase			(							__snake_case					):
      __lowerCamelCase							=					'en'
      __lowerCamelCase							=					'ROMANCE'
      __lowerCamelCase							=					[
          '>>fr<< Don\'t spend so much time watching TV.',
          '>>pt<< Your message has been sent.',
          '>>es<< He\'s two years older than me.',
      ]
      __lowerCamelCase							=					[
          'Ne passez pas autant de temps à regarder la télé.',
          'A sua mensagem foi enviada.',
          'Es dos años más viejo que yo.',
      ]






      @unittest.skip("""Skipping until #12647 is resolved."""						)
      @slow
      def 			lowerCAmelCase_   (      self						)     ->				Dict:


          '''simple docstring'''







          self._assert_generated_batch_equal_expected()






      @unittest.skip("""Skipping until #12647 is resolved."""						)
      @slow
      def 			lowerCAmelCase_   (      self						)     ->				int:


          '''simple docstring'''







          snake_case:						List[str]         =   TranslationPipeline(self.model  ,		self.tokenizer  ,		framework="""tf"""						)
          snake_case:						List[Any]         =   pipeline(self.src_text						)
          self.assertEqual(self.expected_text  ,		[x["""translation_text"""] for x in output]						)


