



'''simple docstring'''


import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
      RwkvForCausalLM,
      RwkvModel,
  )
  from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
  lowercase											=      False







class 						UpperCAmelCase						:


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_=1_4				,	lowerCAmelCase_=7				,	lowerCAmelCase_=True				,	lowerCAmelCase_=False				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=9_9				,	lowerCAmelCase_=3_2				,	lowerCAmelCase_=5				,	lowerCAmelCase_=3_7				,	lowerCAmelCase_="gelu"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=5_1_2				,	lowerCAmelCase_=1_6				,	lowerCAmelCase_=2				,	lowerCAmelCase_=3				,	lowerCAmelCase_=4				,	lowerCAmelCase_=None				,	)			->   Any:



      """simple docstring"""







      a_									=parent
      a_									=batch_size
      a_									=seq_length
      a_									=is_training
      a_									=use_token_type_ids
      a_									=use_input_mask
      a_									=use_labels
      a_									=use_mc_token_ids
      a_									=vocab_size
      a_									=hidden_size
      a_									=num_hidden_layers
      a_									=intermediate_size
      a_									=hidden_act
      a_									=hidden_dropout_prob
      a_									=attention_probs_dropout_prob
      a_									=max_position_embeddings
      a_									=type_vocab_size
      a_									=type_sequence_label_size
      a_									=num_labels
      a_									=num_choices
      a_									=scope
      a_									=vocab_size - 1
      a_									=vocab_size - 1
      a_									=vocab_size - 1




 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      return RwkvConfig.from_pretrained("sgugger/rwkv-4-pile-7b")




 def    lowercase_	( self				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False)			->   int:



      """simple docstring"""







      a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.vocab_size)

      a_									=None
      if self.use_input_mask:
           a_									=random_attention_mask([self.batch_size, self.seq_length])

      a_									=None
      if self.use_token_type_ids:
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.type_vocab_size)

      a_									=None
      if self.use_mc_token_ids:
           a_									=ids_tensor([self.batch_size, self.num_choices]				,	self.seq_length)

      a_									=None
      a_									=None
      a_									=None
      if self.use_labels:
           a_									=ids_tensor([self.batch_size]				,	self.type_sequence_label_size)
           a_									=ids_tensor([self.batch_size, self.seq_length]				,	self.num_labels)
           a_									=ids_tensor([self.batch_size]				,	self.num_choices)

      a_									=self.get_config(
          gradient_checkpointing=lowerCAmelCase_				,	scale_attn_by_inverse_layer_idx=lowerCAmelCase_				,	reorder_and_upcast_attn=lowerCAmelCase_				,	)

      return (
          config,
          input_ids,
          input_mask,
          None,
          token_type_ids,
          mc_token_ids,
          sequence_labels,
          token_labels,
          choice_labels,
      )




 def    lowercase_	( self				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False)			->   Optional[int]:



      """simple docstring"""







      return RwkvConfig(
          vocab_size=self.vocab_size				,	hidden_size=self.hidden_size				,	num_hidden_layers=self.num_hidden_layers				,	intermediate_size=self.intermediate_size				,	activation_function=self.hidden_act				,	resid_pdrop=self.hidden_dropout_prob				,	attn_pdrop=self.attention_probs_dropout_prob				,	n_positions=self.max_position_embeddings				,	type_vocab_size=self.type_vocab_size				,	use_cache=lowerCAmelCase_				,	bos_token_id=self.bos_token_id				,	eos_token_id=self.eos_token_id				,	pad_token_id=self.pad_token_id				,	gradient_checkpointing=lowerCAmelCase_				,	scale_attn_by_inverse_layer_idx=lowerCAmelCase_				,	reorder_and_upcast_attn=lowerCAmelCase_				,	)




 def    lowercase_	( self)			->   List[Any]:



      """simple docstring"""







      a_									=self.get_config()
      a_									=3_0_0
      return config




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      (
          (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    
      )									=self.prepare_config_and_inputs()

      a_									=floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
      a_									=ids_tensor([self.batch_size, self.seq_length]				,	vocab_size=2)

      return (
          config,
          input_ids,
          input_mask,
          head_mask,
          token_type_ids,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	*lowerCAmelCase_)			->   Optional[Any]:



      """simple docstring"""







      a_									=True
      a_									=RwkvModel(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()

      a_									=model(lowerCAmelCase_)

      self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, self.seq_length, self.hidden_size))
      self.parent.assertEqual(len(result.hidden_states)				,	config.num_hidden_layers + 1)




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	*lowerCAmelCase_)			->   List[Any]:



      """simple docstring"""







      a_									=RwkvForCausalLM(lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()

      a_									=model(lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.loss.shape				,	())
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length, self.vocab_size))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	*lowerCAmelCase_)			->   Tuple:



      """simple docstring"""







      a_									=RwkvModel(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()

      a_									=model(lowerCAmelCase_)
      a_									=outputs.last_hidden_state

      a_									=model(input_ids[:, :2])
      a_									=outputs.last_hidden_state

      # Using the state computed on the first inputs, we will get the same output
      a_									=model(input_ids[:, 2:]				,	state=outputs.state)
      a_									=outputs.last_hidden_state

      self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]				,	dim=1)				,	lowerCAmelCase_				,	atol=1e-5))




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	*lowerCAmelCase_				,	lowerCAmelCase_=False)			->   Dict:



      """simple docstring"""







      a_									=RwkvForCausalLM(lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      if gradient_checkpointing:
           model.gradient_checkpointing_enable()

      a_									=model(lowerCAmelCase_				,	labels=lowerCAmelCase_)
      self.parent.assertEqual(result.loss.shape				,	())
      self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length, self.vocab_size))
      result.loss.backward()




 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      a_									=self.prepare_config_and_inputs()

      (
          (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    (
          a_
      )		,    
      )									=config_and_inputs

      a_									={"input_ids": input_ids}

      return config, inputs_dict







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0       ,						reason="See https://github.com/huggingface/transformers/pull/24204")
@require_torch
class 						UpperCAmelCase						(		__a       ,						__a       ,						__a       ,						unittest.TestCase):


 '''simple docstring'''




 __magic_name__       :			str	=   (RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
 __magic_name__       :			Optional[int]	=   (
     {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {}
 )
 # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
 __magic_name__       :			str	=   False
 __magic_name__       :			List[Any]	=   False
 __magic_name__       :			Tuple	=   False
 __magic_name__       :			List[Any]	=   False
 __magic_name__       :			Tuple	=   False  # Rwkv does not support head masking




 def    lowercase_	( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=RwkvModelTester(self)
      a_									=ConfigTester(
          self				,	config_class=lowerCAmelCase_				,	n_embd=3_7				,	common_properties=["hidden_size", "num_hidden_layers"])




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_=None)			->   Any:



      """simple docstring"""







      if isinstance(lowerCAmelCase_				,	torch.Tensor):
           a_		,    a_									=member.max().item(), member.min().item()
      elif isinstance(lowerCAmelCase_				,	lowerCAmelCase_) or isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
           a_		,    a_									=max(lowerCAmelCase_), min(lowerCAmelCase_)

      if not isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
           raise TypeError("container should be a list or tuple")
      elif len(lowerCAmelCase_) != 2:
           raise ValueError("container should have 2 elements")

      a_		,    a_									=container

      a_									=(min_value >= expected_min) and (max_value <= expected_max)

      if not is_inside_interval:
           a_									="%s not found in %s" % (safe_repr(lowerCAmelCase_), safe_repr(lowerCAmelCase_))
           self.fail(self._formatMessage(lowerCAmelCase_				,	lowerCAmelCase_))




 def    lowercase_	( self)			->   int:



      """simple docstring"""







      self.config_tester.run_common_tests()




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_rwkv_model(*lowerCAmelCase_)




 def    lowercase_	( self)			->   str:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_causl_lm(*lowerCAmelCase_)




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      a_									=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_state_equivalency(*lowerCAmelCase_)




 def    lowercase_	( self)			->   List[Any]:



      """simple docstring"""







      a_		,    a_									=self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
           a_									=model_class(config=lowerCAmelCase_)
           for name, param in model.named_parameters():
                if "time_decay" in name:
                     if param.requires_grad:
                          self.assertTrue(param.data.max().item() == 3.0)
                          self.assertTrue(param.data.min().item() == -5.0)
                elif "time_first" in name:
                     if param.requires_grad:
                          # check if it's a ones like
                          self.assertTrue(torch.allclose(param.data				,	torch.ones_like(param.data)				,	atol=1e-5				,	rtol=1e-5))
                elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]):
                     if param.requires_grad:
                          self.assertInterval(
                              param.data				,	[0.0, 1.0]				,	msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""				,	)
                elif "time_mix_value" in name:
                     if param.requires_grad:
                          self.assertInterval(
                              param.data				,	[0.0, 1.3]				,	msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""				,	)




 def    lowercase_	( self)			->   Optional[int]:



      """simple docstring"""







      a_		,    a_									=self.model_tester.prepare_config_and_inputs_for_common()
      a_									=True

      a_									=getattr(self.model_tester				,	"seq_length"				,	lowerCAmelCase_)

      for model_class in self.all_model_classes:
           a_									=True
           a_									=False
           a_									=True
           a_									=model_class(lowerCAmelCase_)
           model.to(lowerCAmelCase_)
           model.eval()

           a_									=self._prepare_for_class(lowerCAmelCase_				,	lowerCAmelCase_)
           a_									=inputs["input_ids"].shape[0]
           with torch.no_grad():
                a_									=model(**lowerCAmelCase_)
           a_									=outputs.attentions
           self.assertEqual(len(lowerCAmelCase_)				,	self.model_tester.num_hidden_layers)

           # check that output_attentions also work using config
           del inputs_dict["output_attentions"]
           a_									=True
           a_									=model_class(lowerCAmelCase_)
           model.to(lowerCAmelCase_)
           model.eval()

           a_									=self._prepare_for_class(lowerCAmelCase_				,	lowerCAmelCase_)
           a_									=inputs["input_ids"].shape[0]
           with torch.no_grad():
                a_									=model(**lowerCAmelCase_)
           a_									=outputs.attentions
           self.assertEqual(len(lowerCAmelCase_)				,	self.model_tester.num_hidden_layers)

           self.assertListEqual(
               list(attentions[0].shape[-3:])				,	[batch_size, seq_len, config.hidden_size]				,	)
           a_									=len(lowerCAmelCase_)

           # Check attention is always last and order is fine
           a_									=True
           a_									=True
           a_									=model_class(lowerCAmelCase_)
           model.to(lowerCAmelCase_)
           model.eval()

           a_									=self._prepare_for_class(lowerCAmelCase_				,	lowerCAmelCase_)
           a_									=inputs["input_ids"].shape[0]
           with torch.no_grad():
                a_									=model(**lowerCAmelCase_)

           a_									=1
           self.assertEqual(out_len + added_hidden_states				,	len(lowerCAmelCase_))

           a_									=outputs.attentions

           self.assertEqual(len(lowerCAmelCase_)				,	self.model_tester.num_hidden_layers)
           self.assertListEqual(
               list(self_attentions[0].shape[-3:])				,	[batch_size, seq_len, config.hidden_size]				,	)




 @slow
 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           a_									=RwkvModel.from_pretrained(lowerCAmelCase_)
           self.assertIsNotNone(lowerCAmelCase_)







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0       ,						reason="See https://github.com/huggingface/transformers/pull/24204")
@slow
class 						UpperCAmelCase						(		unittest.TestCase):


 '''simple docstring'''




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      a_									="RWKV/rwkv-4-169m-pile"
      a_									=AutoTokenizer.from_pretrained(self.model_id)




 def    lowercase_	( self)			->   List[str]:



      """simple docstring"""







      a_									="Hello my name is Jasmine and I am a newbie to the"
      a_									=RwkvForCausalLM.from_pretrained(self.model_id).to(lowerCAmelCase_)

      a_									=self.tokenizer("Hello my name is"				,	return_tensors="pt").input_ids.to(lowerCAmelCase_)
      a_									=model.generate(lowerCAmelCase_				,	max_new_tokens=1_0)
      a_									=self.tokenizer.decode(output[0].tolist())

      self.assertEqual(lowerCAmelCase_				,	lowerCAmelCase_)




 def    lowercase_	( self)			->   Union[str, Any]:



      """simple docstring"""







      a_									="Hello my name is Jasmine and I am a newbie to the"

      a_									=self.tokenizer("Hello my name is"				,	return_tensors="pt").input_ids.to(lowerCAmelCase_)
      a_									=RwkvForCausalLM.from_pretrained(self.model_id				,	torch_dtype=torch.bfloataa).to(lowerCAmelCase_)

      a_									=model.generate(lowerCAmelCase_				,	max_new_tokens=1_0)
      a_									=self.tokenizer.decode(output[0].tolist())

      self.assertEqual(lowerCAmelCase_				,	lowerCAmelCase_)







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase											=      {
    '''configuration_align''': [
        '''ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AlignConfig''',
        '''AlignTextConfig''',
        '''AlignVisionConfig''',
    ],
    '''processing_align''': ['''AlignProcessor'''],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase											=      [
      '''ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''AlignModel''',
      '''AlignPreTrainedModel''',
      '''AlignTextModel''',
      '''AlignVisionModel''',
  ]

if TYPE_CHECKING:
  from .configuration_align import (
      ALIGN_PRETRAINED_CONFIG_ARCHIVE_MAP,
      AlignConfig,
      AlignTextConfig,
      AlignVisionConfig,
  )
  from .processing_align import AlignProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_align import (
        ALIGN_PRETRAINED_MODEL_ARCHIVE_LIST,
        AlignModel,
        AlignPreTrainedModel,
        AlignTextModel,
        AlignVisionModel,
    )

else:
  import sys

  lowercase											=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


