






import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
     RwkvForCausalLM,
     RwkvModel,
 )
 from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
 a 			=							False
class 							__snake_case:



 def __init__(						self : int			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Optional[int]=14			,		UpperCAmelCase__ : Optional[Any]=7			,		UpperCAmelCase__ : Dict=True			,		UpperCAmelCase__ : Dict=False			,		UpperCAmelCase__ : str=True			,		UpperCAmelCase__ : Dict=True			,		UpperCAmelCase__ : Tuple=True			,		UpperCAmelCase__ : str=99			,		UpperCAmelCase__ : Union[str, Any]=32			,		UpperCAmelCase__ : Optional[int]=5			,		UpperCAmelCase__ : Optional[Any]=37			,		UpperCAmelCase__ : Union[str, Any]="gelu"			,		UpperCAmelCase__ : Dict=0.1			,		UpperCAmelCase__ : Any=0.1			,		UpperCAmelCase__ : Tuple=512			,		UpperCAmelCase__ : Optional[int]=16			,		UpperCAmelCase__ : str=2			,		UpperCAmelCase__ : Dict=3			,		UpperCAmelCase__ : List[Any]=4			,		UpperCAmelCase__ : str=None			,		):

    """simple docstring"""






    lowercase__    :						Optional[int]       =			parent
    lowercase__    :						Dict       =			batch_size
    lowercase__    :						Union[str, Any]       =			seq_length
    lowercase__    :						Dict       =			is_training
    lowercase__    :						Union[str, Any]       =			use_token_type_ids
    lowercase__    :						Dict       =			use_input_mask
    lowercase__    :						Optional[int]       =			use_labels
    lowercase__    :						List[Any]       =			use_mc_token_ids
    lowercase__    :						List[Any]       =			vocab_size
    lowercase__    :						str       =			hidden_size
    lowercase__    :						Optional[int]       =			num_hidden_layers
    lowercase__    :						Union[str, Any]       =			intermediate_size
    lowercase__    :						Optional[Any]       =			hidden_act
    lowercase__    :						Dict       =			hidden_dropout_prob
    lowercase__    :						List[str]       =			attention_probs_dropout_prob
    lowercase__    :						Union[str, Any]       =			max_position_embeddings
    lowercase__    :						Optional[Any]       =			type_vocab_size
    lowercase__    :						Union[str, Any]       =			type_sequence_label_size
    lowercase__    :						List[Any]       =			num_labels
    lowercase__    :						Tuple       =			num_choices
    lowercase__    :						List[str]       =			scope
    lowercase__    :						List[Any]       =			vocab_size - 1
    lowercase__    :						int       =			vocab_size - 1
    lowercase__    :						Any       =			vocab_size - 1



 def     UpperCAmelCase     (						self : int						):

    """simple docstring"""






    return RwkvConfig.from_pretrained("""sgugger/rwkv-4-pile-7b"""						)



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : Optional[Any]=False			,		UpperCAmelCase__ : List[str]=False			,		UpperCAmelCase__ : Optional[Any]=False						):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			ids_tensor([self.batch_size, self.seq_length]			,		self.vocab_size						)

    lowercase__    :						List[str]       =			None
    if self.use_input_mask:
       lowercase__    :						List[str]       =			random_attention_mask([self.batch_size, self.seq_length]						)

    lowercase__    :						int       =			None
    if self.use_token_type_ids:
       lowercase__    :						str       =			ids_tensor([self.batch_size, self.seq_length]			,		self.type_vocab_size						)

    lowercase__    :						Dict       =			None
    if self.use_mc_token_ids:
       lowercase__    :						int       =			ids_tensor([self.batch_size, self.num_choices]			,		self.seq_length						)

    lowercase__    :						List[str]       =			None
    lowercase__    :						int       =			None
    lowercase__    :						Optional[int]       =			None
    if self.use_labels:
       lowercase__    :						str       =			ids_tensor([self.batch_size]			,		self.type_sequence_label_size						)
       lowercase__    :						str       =			ids_tensor([self.batch_size, self.seq_length]			,		self.num_labels						)
       lowercase__    :						Dict       =			ids_tensor([self.batch_size]			,		self.num_choices						)

    lowercase__    :						Any       =			self.get_config(
        gradient_checkpointing=UpperCAmelCase__			,		scale_attn_by_inverse_layer_idx=UpperCAmelCase__			,		reorder_and_upcast_attn=UpperCAmelCase__			,		)

    return (
        config,
        input_ids,
        input_mask,
        None,
        token_type_ids,
        mc_token_ids,
        sequence_labels,
        token_labels,
        choice_labels,
    )



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : Any=False			,		UpperCAmelCase__ : Optional[Any]=False			,		UpperCAmelCase__ : Optional[Any]=False						):

    """simple docstring"""






    return RwkvConfig(
        vocab_size=self.vocab_size			,		hidden_size=self.hidden_size			,		num_hidden_layers=self.num_hidden_layers			,		intermediate_size=self.intermediate_size			,		activation_function=self.hidden_act			,		resid_pdrop=self.hidden_dropout_prob			,		attn_pdrop=self.attention_probs_dropout_prob			,		n_positions=self.max_position_embeddings			,		type_vocab_size=self.type_vocab_size			,		use_cache=UpperCAmelCase__			,		bos_token_id=self.bos_token_id			,		eos_token_id=self.eos_token_id			,		pad_token_id=self.pad_token_id			,		gradient_checkpointing=UpperCAmelCase__			,		scale_attn_by_inverse_layer_idx=UpperCAmelCase__			,		reorder_and_upcast_attn=UpperCAmelCase__			,		)



 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    lowercase__    :						Optional[Any]       =			self.get_config()
    lowercase__    :						Optional[int]       =			300
    return config



 def     UpperCAmelCase     (						self : List[Any]						):

    """simple docstring"""






    (
        (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       
    )    :						List[Any]       =			self.prepare_config_and_inputs()

    lowercase__    :						Optional[Any]       =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
    lowercase__    :						List[Any]       =			ids_tensor([self.batch_size, self.seq_length]			,		vocab_size=2						)

    return (
        config,
        input_ids,
        input_mask,
        head_mask,
        token_type_ids,
        sequence_labels,
        token_labels,
        choice_labels,
        encoder_hidden_states,
        encoder_attention_mask,
    )



 def     UpperCAmelCase     (						self : Any			,		UpperCAmelCase__ : Optional[int]			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Optional[int]			,		*UpperCAmelCase__ : int						):

    """simple docstring"""






    lowercase__    :						str       =			True
    lowercase__    :						List[Any]       =			RwkvModel(config=UpperCAmelCase__						)
    model.to(UpperCAmelCase__						)
    model.eval()

    lowercase__    :						int       =			model(UpperCAmelCase__						)

    self.parent.assertEqual(result.last_hidden_state.shape			,		(self.batch_size, self.seq_length, self.hidden_size)						)
    self.parent.assertEqual(len(result.hidden_states						)			,		config.num_hidden_layers + 1						)



 def     UpperCAmelCase     (						self : Dict			,		UpperCAmelCase__ : List[Any]			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : int			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : List[str]			,		*UpperCAmelCase__ : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						str       =			RwkvForCausalLM(UpperCAmelCase__						)
    model.to(UpperCAmelCase__						)
    model.eval()

    lowercase__    :						Dict       =			model(UpperCAmelCase__			,		labels=UpperCAmelCase__						)
    self.parent.assertEqual(result.loss.shape			,		()						)
    self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.vocab_size)						)



 def     UpperCAmelCase     (						self : List[str]			,		UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : List[str]			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : Tuple			,		*UpperCAmelCase__ : List[str]						):

    """simple docstring"""






    lowercase__    :						Any       =			RwkvModel(config=UpperCAmelCase__						)
    model.to(UpperCAmelCase__						)
    model.eval()

    lowercase__    :						Optional[int]       =			model(UpperCAmelCase__						)
    lowercase__    :						Optional[int]       =			outputs.last_hidden_state

    lowercase__    :						Optional[Any]       =			model(input_ids[:, :2]						)
    lowercase__    :						List[str]       =			outputs.last_hidden_state

    # Using the state computed on the first inputs, we will get the same output
    lowercase__    :						Optional[int]       =			model(input_ids[:, 2:]			,		state=outputs.state						)
    lowercase__    :						List[str]       =			outputs.last_hidden_state

    self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]			,		dim=1						)			,		UpperCAmelCase__			,		atol=1e-5						)						)



 def     UpperCAmelCase     (						self : Optional[int]			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : Optional[int]			,		UpperCAmelCase__ : Dict			,		UpperCAmelCase__ : int			,		UpperCAmelCase__ : Tuple			,		*UpperCAmelCase__ : Union[str, Any]			,		UpperCAmelCase__ : Tuple=False						):

    """simple docstring"""






    lowercase__    :						str       =			RwkvForCausalLM(UpperCAmelCase__						)
    model.to(UpperCAmelCase__						)
    if gradient_checkpointing:
       model.gradient_checkpointing_enable()

    lowercase__    :						Union[str, Any]       =			model(UpperCAmelCase__			,		labels=UpperCAmelCase__						)
    self.parent.assertEqual(result.loss.shape			,		()						)
    self.parent.assertEqual(result.logits.shape			,		(self.batch_size, self.seq_length, self.vocab_size)						)
    result.loss.backward()







 def     UpperCAmelCase     (						self : int						):

    """simple docstring"""






    lowercase__    :						List[Any]       =			self.prepare_config_and_inputs()

    (
        (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       (
        lowercase__
    )  ,       
    )    :						Optional[Any]       =			config_and_inputs

    lowercase__    :						List[str]       =			{"""input_ids""": input_ids}

    return config, inputs_dict
@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0  ,							reason='''See https://github.com/huggingface/transformers/pull/24204'''   )
@require_torch
class 							__snake_case(       __UpperCamelCase  ,							__UpperCamelCase  ,							__UpperCamelCase  ,							unittest.TestCase   ):
 lowercase			:							Union[str, Any]		   =		(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
 lowercase			:							List[Any]		   =		(
     {'''feature-extraction''': RwkvModel, '''text-generation''': RwkvForCausalLM} if is_torch_available() else {}
 )
 # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
 lowercase			:							Tuple		   =		False
 lowercase			:							List[str]		   =		False
 lowercase			:							Any		   =		False
 lowercase			:							int		   =		False
 lowercase			:							Dict		   =		False  # Rwkv does not support head masking



 def     UpperCAmelCase     (						self : Any						):

    """simple docstring"""






    lowercase__    :						Dict       =			RwkvModelTester(self						)
    lowercase__    :						str       =			ConfigTester(
        self			,		config_class=UpperCAmelCase__			,		n_embd=37			,		common_properties=["""hidden_size""", """num_hidden_layers"""]						)



 def     UpperCAmelCase     (						self : Any			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Optional[Any]			,		UpperCAmelCase__ : Dict=None						):

    """simple docstring"""






    if isinstance(UpperCAmelCase__			,		torch.Tensor						):
       lowercase__  ,       lowercase__    :						List[str]       =			member.max().item(), member.min().item()
    elif isinstance(UpperCAmelCase__			,		UpperCAmelCase__						) or isinstance(UpperCAmelCase__			,		UpperCAmelCase__						):
       lowercase__  ,       lowercase__    :						List[str]       =			max(UpperCAmelCase__						), min(UpperCAmelCase__						)

    if not isinstance(UpperCAmelCase__			,		UpperCAmelCase__						):
       raise TypeError("""container should be a list or tuple"""						)
    elif len(UpperCAmelCase__						) != 2:
       raise ValueError("""container should have 2 elements"""						)

    lowercase__  ,       lowercase__    :						Tuple       =			container

    lowercase__    :						Dict       =			(min_value >= expected_min) and (max_value <= expected_max)

    if not is_inside_interval:
       lowercase__    :						List[Any]       =			"""%s not found in %s""" % (safe_repr(UpperCAmelCase__						), safe_repr(UpperCAmelCase__						))
       self.fail(self._formatMessage(UpperCAmelCase__			,		UpperCAmelCase__						)						)



 def     UpperCAmelCase     (						self : List[Any]						):

    """simple docstring"""






    self.config_tester.run_common_tests()



 def     UpperCAmelCase     (						self : Optional[int]						):

    """simple docstring"""






    lowercase__    :						int       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_rwkv_model(*UpperCAmelCase__						)



 def     UpperCAmelCase     (						self : Optional[int]						):

    """simple docstring"""






    lowercase__    :						Tuple       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_causl_lm(*UpperCAmelCase__						)



 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    lowercase__    :						List[str]       =			self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_state_equivalency(*UpperCAmelCase__						)



 def     UpperCAmelCase     (						self : List[str]						):

    """simple docstring"""






    lowercase__  ,       lowercase__    :						Optional[Any]       =			self.model_tester.prepare_config_and_inputs_for_common()

    for model_class in self.all_model_classes:
       lowercase__    :						Union[str, Any]       =			model_class(config=UpperCAmelCase__						)
       for name, param in model.named_parameters():
          if "time_decay" in name:
             if param.requires_grad:
                self.assertTrue(param.data.max().item() == 3.0						)
                self.assertTrue(param.data.min().item() == -5.0						)
          elif "time_first" in name:
             if param.requires_grad:
                # check if it's a ones like
                self.assertTrue(torch.allclose(param.data			,		torch.ones_like(param.data						)			,		atol=1e-5			,		rtol=1e-5						)						)
          elif any(x in name for x in ["""time_mix_key""", """time_mix_receptance"""]						):
             if param.requires_grad:
                self.assertInterval(
                    param.data			,		[0.0, 1.0]			,		msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''			,		)
          elif "time_mix_value" in name:
             if param.requires_grad:
                self.assertInterval(
                    param.data			,		[0.0, 1.3]			,		msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''			,		)



 def     UpperCAmelCase     (						self : str						):

    """simple docstring"""






    lowercase__  ,       lowercase__    :						Tuple       =			self.model_tester.prepare_config_and_inputs_for_common()
    lowercase__    :						Any       =			True

    lowercase__    :						Optional[int]       =			getattr(self.model_tester			,		"""seq_length"""			,		UpperCAmelCase__						)

    for model_class in self.all_model_classes:
       lowercase__    :						Dict       =			True
       lowercase__    :						int       =			False
       lowercase__    :						Any       =			True
       lowercase__    :						Optional[int]       =			model_class(UpperCAmelCase__						)
       model.to(UpperCAmelCase__						)
       model.eval()

       lowercase__    :						Optional[Any]       =			self._prepare_for_class(UpperCAmelCase__			,		UpperCAmelCase__						)
       lowercase__    :						Dict       =			inputs["""input_ids"""].shape[0]
       with torch.no_grad():
          lowercase__    :						Optional[int]       =			model(**UpperCAmelCase__						)
       lowercase__    :						int       =			outputs.attentions
       self.assertEqual(len(UpperCAmelCase__						)			,		self.model_tester.num_hidden_layers						)

       # check that output_attentions also work using config
       del inputs_dict["output_attentions"]
       lowercase__    :						Any       =			True
       lowercase__    :						Optional[Any]       =			model_class(UpperCAmelCase__						)
       model.to(UpperCAmelCase__						)
       model.eval()

       lowercase__    :						List[str]       =			self._prepare_for_class(UpperCAmelCase__			,		UpperCAmelCase__						)
       lowercase__    :						Optional[int]       =			inputs["""input_ids"""].shape[0]
       with torch.no_grad():
          lowercase__    :						str       =			model(**UpperCAmelCase__						)
       lowercase__    :						str       =			outputs.attentions
       self.assertEqual(len(UpperCAmelCase__						)			,		self.model_tester.num_hidden_layers						)

       self.assertListEqual(
           list(attentions[0].shape[-3:]						)			,		[batch_size, seq_len, config.hidden_size]			,		)
       lowercase__    :						Tuple       =			len(UpperCAmelCase__						)

       # Check attention is always last and order is fine
       lowercase__    :						Union[str, Any]       =			True
       lowercase__    :						Optional[Any]       =			True
       lowercase__    :						Any       =			model_class(UpperCAmelCase__						)
       model.to(UpperCAmelCase__						)
       model.eval()

       lowercase__    :						Optional[int]       =			self._prepare_for_class(UpperCAmelCase__			,		UpperCAmelCase__						)
       lowercase__    :						List[Any]       =			inputs["""input_ids"""].shape[0]
       with torch.no_grad():
          lowercase__    :						int       =			model(**UpperCAmelCase__						)

       lowercase__    :						List[str]       =			1
       self.assertEqual(out_len + added_hidden_states			,		len(UpperCAmelCase__						)						)

       lowercase__    :						List[str]       =			outputs.attentions

       self.assertEqual(len(UpperCAmelCase__						)			,		self.model_tester.num_hidden_layers						)
       self.assertListEqual(
           list(self_attentions[0].shape[-3:]						)			,		[batch_size, seq_len, config.hidden_size]			,		)







 @slow
 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       lowercase__    :						Any       =			RwkvModel.from_pretrained(UpperCAmelCase__						)
       self.assertIsNotNone(UpperCAmelCase__						)
@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0  ,							reason='''See https://github.com/huggingface/transformers/pull/24204'''   )
@slow
class 							__snake_case(       unittest.TestCase   ):



 def     UpperCAmelCase     (						self : Optional[Any]						):

    """simple docstring"""






    lowercase__    :						List[str]       =			"""RWKV/rwkv-4-169m-pile"""
    lowercase__    :						Optional[int]       =			AutoTokenizer.from_pretrained(self.model_id						)



 def     UpperCAmelCase     (						self : List[str]						):

    """simple docstring"""






    lowercase__    :						List[str]       =			"""Hello my name is Jasmine and I am a newbie to the"""
    lowercase__    :						Any       =			RwkvForCausalLM.from_pretrained(self.model_id						).to(UpperCAmelCase__						)

    lowercase__    :						List[Any]       =			self.tokenizer("""Hello my name is"""			,		return_tensors="""pt"""						).input_ids.to(UpperCAmelCase__						)
    lowercase__    :						Any       =			model.generate(UpperCAmelCase__			,		max_new_tokens=10						)
    lowercase__    :						Union[str, Any]       =			self.tokenizer.decode(output[0].tolist()						)

    self.assertEqual(UpperCAmelCase__			,		UpperCAmelCase__						)







 def     UpperCAmelCase     (						self : List[str]						):

    """simple docstring"""






    lowercase__    :						List[Any]       =			"""Hello my name is Jasmine and I am a newbie to the"""

    lowercase__    :						Tuple       =			self.tokenizer("""Hello my name is"""			,		return_tensors="""pt"""						).input_ids.to(UpperCAmelCase__						)
    lowercase__    :						Optional[Any]       =			RwkvForCausalLM.from_pretrained(self.model_id			,		torch_dtype=torch.bfloataa						).to(UpperCAmelCase__						)

    lowercase__    :						Any       =			model.generate(UpperCAmelCase__			,		max_new_tokens=10						)
    lowercase__    :						Tuple       =			self.tokenizer.decode(output[0].tolist()						)

    self.assertEqual(UpperCAmelCase__			,		UpperCAmelCase__						)










from __future__ import annotations

from typing import Any
class 							__snake_case:



 def __init__(						self : str			,		UpperCAmelCase__ : Any			,		UpperCAmelCase__ : Any						):  # noqa: A002

    """simple docstring"""






    lowercase__    :						Optional[int]       =			item
    lowercase__    :						Union[str, Any]       =			next
class 							__snake_case:



 def __init__(						self : List[Any]						):

    """simple docstring"""






    lowercase__    :						Node | None       =			None
    lowercase__    :						Optional[Any]       =			0



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : Any						):

    """simple docstring"""






    lowercase__    :						Any       =			Node(UpperCAmelCase__			,		self.head						)
    self.size += 1



 def     UpperCAmelCase     (						self : Dict						):

    """simple docstring"""






    if self.head is None:
       return None
    else:
       lowercase__    :						List[Any]       =			self.head.item
       lowercase__    :						Optional[Any]       =			self.head.next
       self.size -= 1
       return item



 def     UpperCAmelCase     (						self : Union[str, Any]						):

    """simple docstring"""






    return self.head is None



 def __str__(						self : List[str]						):

    """simple docstring"""






    if self.is_empty():
       return ""
    else:
       lowercase__    :						Optional[Any]       =			self.head
       lowercase__    :						str       =			""""""
       lowercase__    :						list[str]       =			[]
       while iterate:
          item_list.append(str(iterate.item						)						)
          lowercase__    :						List[str]       =			iterate.next

       lowercase__    :						List[str]       =			""" --> """.join(UpperCAmelCase__						)

       return item_str







 def __len__(						self : List[Any]						):

    """simple docstring"""






    return self.size


