






import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
          RwkvForCausalLM,
          RwkvModel,
      )
      from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
      __UpperCAmelCase      =				False
class 	a_:



     """simple docstring"""



     def __init__(		self					:	Any  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	Dict=1_4  ,  lowerCAmelCase__					:	str=7  ,  lowerCAmelCase__					:	List[Any]=True  ,  lowerCAmelCase__					:	Optional[int]=False  ,  lowerCAmelCase__					:	List[str]=True  ,  lowerCAmelCase__					:	Optional[int]=True  ,  lowerCAmelCase__					:	Optional[int]=True  ,  lowerCAmelCase__					:	Union[str, Any]=9_9  ,  lowerCAmelCase__					:	Optional[Any]=3_2  ,  lowerCAmelCase__					:	List[Any]=5  ,  lowerCAmelCase__					:	str=3_7  ,  lowerCAmelCase__					:	str="gelu"  ,  lowerCAmelCase__					:	Optional[Any]=0.1  ,  lowerCAmelCase__					:	List[Any]=0.1  ,  lowerCAmelCase__					:	Optional[Any]=5_1_2  ,  lowerCAmelCase__					:	Optional[Any]=1_6  ,  lowerCAmelCase__					:	Optional[Any]=2  ,  lowerCAmelCase__					:	str=3  ,  lowerCAmelCase__					:	str=4  ,  lowerCAmelCase__					:	Union[str, Any]=None  ,  )							->	Optional[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		parent
      SCREAMING_SNAKE_CASE				    =		batch_size
      SCREAMING_SNAKE_CASE				    =		seq_length
      SCREAMING_SNAKE_CASE				    =		is_training
      SCREAMING_SNAKE_CASE				    =		use_token_type_ids
      SCREAMING_SNAKE_CASE				    =		use_input_mask
      SCREAMING_SNAKE_CASE				    =		use_labels
      SCREAMING_SNAKE_CASE				    =		use_mc_token_ids
      SCREAMING_SNAKE_CASE				    =		vocab_size
      SCREAMING_SNAKE_CASE				    =		hidden_size
      SCREAMING_SNAKE_CASE				    =		num_hidden_layers
      SCREAMING_SNAKE_CASE				    =		intermediate_size
      SCREAMING_SNAKE_CASE				    =		hidden_act
      SCREAMING_SNAKE_CASE				    =		hidden_dropout_prob
      SCREAMING_SNAKE_CASE				    =		attention_probs_dropout_prob
      SCREAMING_SNAKE_CASE				    =		max_position_embeddings
      SCREAMING_SNAKE_CASE				    =		type_vocab_size
      SCREAMING_SNAKE_CASE				    =		type_sequence_label_size
      SCREAMING_SNAKE_CASE				    =		num_labels
      SCREAMING_SNAKE_CASE				    =		num_choices
      SCREAMING_SNAKE_CASE				    =		scope
      SCREAMING_SNAKE_CASE				    =		vocab_size - 1
      SCREAMING_SNAKE_CASE				    =		vocab_size - 1
      SCREAMING_SNAKE_CASE				    =		vocab_size - 1





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	Tuple:

      """simple docstring"""
      return RwkvConfig.from_pretrained('sgugger/rwkv-4-pile-7b')





     def     __UpperCamelCase						(		self					:	Tuple  ,  lowerCAmelCase__					:	int=False  ,  lowerCAmelCase__					:	List[Any]=False  ,  lowerCAmelCase__					:	Optional[int]=False)							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size, self.seq_length]  ,  self.vocab_size)

      SCREAMING_SNAKE_CASE				    =		None
      if self.use_input_mask:
       SCREAMING_SNAKE_CASE				    =		random_attention_mask([self.batch_size, self.seq_length])

      SCREAMING_SNAKE_CASE				    =		None
      if self.use_token_type_ids:
       SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size, self.seq_length]  ,  self.type_vocab_size)

      SCREAMING_SNAKE_CASE				    =		None
      if self.use_mc_token_ids:
       SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size, self.num_choices]  ,  self.seq_length)

      SCREAMING_SNAKE_CASE				    =		None
      SCREAMING_SNAKE_CASE				    =		None
      SCREAMING_SNAKE_CASE				    =		None
      if self.use_labels:
       SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size]  ,  self.type_sequence_label_size)
       SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size, self.seq_length]  ,  self.num_labels)
       SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size]  ,  self.num_choices)

      SCREAMING_SNAKE_CASE				    =		self.get_config(
          gradient_checkpointing=a__  ,  scale_attn_by_inverse_layer_idx=a__  ,  reorder_and_upcast_attn=a__  ,  )

      return (
          config,
          input_ids,
          input_mask,
          None,
          token_type_ids,
          mc_token_ids,
          sequence_labels,
          token_labels,
          choice_labels,
      )





     def     __UpperCamelCase						(		self					:	Optional[int]  ,  lowerCAmelCase__					:	List[Any]=False  ,  lowerCAmelCase__					:	Dict=False  ,  lowerCAmelCase__					:	List[Any]=False)							->	Tuple:

      """simple docstring"""
      return RwkvConfig(
          vocab_size=self.vocab_size  ,  hidden_size=self.hidden_size  ,  num_hidden_layers=self.num_hidden_layers  ,  intermediate_size=self.intermediate_size  ,  activation_function=self.hidden_act  ,  resid_pdrop=self.hidden_dropout_prob  ,  attn_pdrop=self.attention_probs_dropout_prob  ,  n_positions=self.max_position_embeddings  ,  type_vocab_size=self.type_vocab_size  ,  use_cache=a__  ,  bos_token_id=self.bos_token_id  ,  eos_token_id=self.eos_token_id  ,  pad_token_id=self.pad_token_id  ,  gradient_checkpointing=a__  ,  scale_attn_by_inverse_layer_idx=a__  ,  reorder_and_upcast_attn=a__  ,  )





     def     __UpperCamelCase						(		self					:	str)							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.get_config()
      SCREAMING_SNAKE_CASE				    =		3_0_0
      return config





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	int:

      """simple docstring"""
      (
          SCREAMING_SNAKE_CASE					
      )				    =		self.prepare_config_and_inputs()

      SCREAMING_SNAKE_CASE				    =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
      SCREAMING_SNAKE_CASE				    =		ids_tensor([self.batch_size, self.seq_length]  ,  vocab_size=2)

      return (
          config,
          input_ids,
          input_mask,
          head_mask,
          token_type_ids,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )





     def     __UpperCamelCase						(		self					:	Optional[Any]  ,  lowerCAmelCase__					:	Union[str, Any]  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	int  ,  lowerCAmelCase__					:	int  ,  lowerCAmelCase__					:	Optional[int]  ,  *lowerCAmelCase__					:	Optional[Any])							->	str:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		True
      SCREAMING_SNAKE_CASE				    =		RwkvModel(config=a__)
      model.to(a__)
      model.eval()

      SCREAMING_SNAKE_CASE				    =		model(a__)

      self.parent.assertEqual(result.last_hidden_state.shape  ,  (self.batch_size, self.seq_length, self.hidden_size))
      self.parent.assertEqual(len(result.hidden_states)  ,  config.num_hidden_layers + 1)





     def     __UpperCamelCase						(		self					:	Any  ,  lowerCAmelCase__					:	Optional[int]  ,  lowerCAmelCase__					:	Optional[int]  ,  lowerCAmelCase__					:	Dict  ,  lowerCAmelCase__					:	Tuple  ,  lowerCAmelCase__					:	Optional[Any]  ,  *lowerCAmelCase__					:	str)							->	Tuple:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		RwkvForCausalLM(a__)
      model.to(a__)
      model.eval()

      SCREAMING_SNAKE_CASE				    =		model(a__  ,  labels=a__)
      self.parent.assertEqual(result.loss.shape  ,  ())
      self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.seq_length, self.vocab_size))





     def     __UpperCamelCase						(		self					:	Tuple  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	List[Any]  ,  lowerCAmelCase__					:	List[str]  ,  lowerCAmelCase__					:	Optional[int]  ,  lowerCAmelCase__					:	str  ,  *lowerCAmelCase__					:	List[Any])							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		RwkvModel(config=a__)
      model.to(a__)
      model.eval()

      SCREAMING_SNAKE_CASE				    =		model(a__)
      SCREAMING_SNAKE_CASE				    =		outputs.last_hidden_state

      SCREAMING_SNAKE_CASE				    =		model(input_ids[:, :2])
      SCREAMING_SNAKE_CASE				    =		outputs.last_hidden_state

      # Using the state computed on the first inputs, we will get the same output
      SCREAMING_SNAKE_CASE				    =		model(input_ids[:, 2:]  ,  state=outputs.state)
      SCREAMING_SNAKE_CASE				    =		outputs.last_hidden_state

      self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]  ,  dim=1)  ,  a__  ,  atol=1e-5))





     def     __UpperCamelCase						(		self					:	List[Any]  ,  lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	List[str]  ,  lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	str  ,  lowerCAmelCase__					:	List[str]  ,  *lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	List[str]=False)							->	str:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		RwkvForCausalLM(a__)
      model.to(a__)
      if gradient_checkpointing:
       model.gradient_checkpointing_enable()

      SCREAMING_SNAKE_CASE				    =		model(a__  ,  labels=a__)
      self.parent.assertEqual(result.loss.shape  ,  ())
      self.parent.assertEqual(result.logits.shape  ,  (self.batch_size, self.seq_length, self.vocab_size))
      result.loss.backward()





     def     __UpperCamelCase						(		self					:	Tuple)							->	Union[str, Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.prepare_config_and_inputs()

      (
          SCREAMING_SNAKE_CASE					
      )				    =		config_and_inputs

      SCREAMING_SNAKE_CASE				    =		{'''input_ids''': input_ids}

      return config, inputs_dict

@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0			,							reason='''See https://github.com/huggingface/transformers/pull/24204'''	)
@require_torch
class 	a_(		UpperCAmelCase_			,							UpperCAmelCase_			,							UpperCAmelCase_			,							unittest.TestCase	):



     """simple docstring"""



     __snake_case		:				Tuple  					=(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
     __snake_case		:				Union[str, Any]  					=(
         {'''feature-extraction''': RwkvModel, '''text-generation''': RwkvForCausalLM} if is_torch_available() else {}
     )
     # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
     __snake_case		:				List[str]  					=False
     __snake_case		:				List[Any]  					=False
     __snake_case		:				Tuple  					=False
     __snake_case		:				Optional[Any]  					=False
     __snake_case		:				int  					=False  # Rwkv does not support head masking





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	str:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		RwkvModelTester(self)
      SCREAMING_SNAKE_CASE				    =		ConfigTester(
          self  ,  config_class=a__  ,  n_embd=3_7  ,  common_properties=['hidden_size', 'num_hidden_layers'])





     def     __UpperCamelCase						(		self					:	int  ,  lowerCAmelCase__					:	Any  ,  lowerCAmelCase__					:	List[str]  ,  lowerCAmelCase__					:	List[str]=None)							->	Tuple:

      """simple docstring"""
      if isinstance(a__  ,  torch.Tensor):
       SCREAMING_SNAKE_CASE				    =		member.max().item(), member.min().item()
      elif isinstance(a__  ,  a__) or isinstance(a__  ,  a__):
       SCREAMING_SNAKE_CASE				    =		max(a__), min(a__)

      if not isinstance(a__  ,  a__):
       raise TypeError('container should be a list or tuple')
      elif len(a__) != 2:
       raise ValueError('container should have 2 elements')

      SCREAMING_SNAKE_CASE				    =		container

      SCREAMING_SNAKE_CASE				    =		(min_value >= expected_min) and (max_value <= expected_max)

      if not is_inside_interval:
       SCREAMING_SNAKE_CASE				    =		'''%s not found in %s''' % (safe_repr(a__), safe_repr(a__))
       self.fail(self._formatMessage(a__  ,  a__))





     def     __UpperCamelCase						(		self					:	Any)							->	Optional[Any]:

      """simple docstring"""
      self.config_tester.run_common_tests()





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	Optional[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_rwkv_model(*a__)





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	Any:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_causl_lm(*a__)





     def     __UpperCamelCase						(		self					:	Union[str, Any])							->	Tuple:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_state_equivalency(*a__)





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	int:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE				    =		model_class(config=a__)
       for name, param in model.named_parameters():
        if "time_decay" in name:
         if param.requires_grad:
          self.assertTrue(param.data.max().item() == 3.0)
          self.assertTrue(param.data.min().item() == -5.0)
        elif "time_first" in name:
         if param.requires_grad:
          # check if it's a ones like
          self.assertTrue(torch.allclose(param.data  ,  torch.ones_like(param.data)  ,  atol=1e-5  ,  rtol=1e-5))
        elif any(x in name for x in ['time_mix_key', 'time_mix_receptance']):
         if param.requires_grad:
          self.assertInterval(
              param.data  ,  [0.0, 1.0]  ,  msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''  ,  )
        elif "time_mix_value" in name:
         if param.requires_grad:
          self.assertInterval(
              param.data  ,  [0.0, 1.3]  ,  msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''  ,  )





     def     __UpperCamelCase						(		self					:	Tuple)							->	str:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		self.model_tester.prepare_config_and_inputs_for_common()
      SCREAMING_SNAKE_CASE				    =		True

      SCREAMING_SNAKE_CASE				    =		getattr(self.model_tester  ,  'seq_length'  ,  a__)

      for model_class in self.all_model_classes:
       SCREAMING_SNAKE_CASE				    =		True
       SCREAMING_SNAKE_CASE				    =		False
       SCREAMING_SNAKE_CASE				    =		True
       SCREAMING_SNAKE_CASE				    =		model_class(a__)
       model.to(a__)
       model.eval()

       SCREAMING_SNAKE_CASE				    =		self._prepare_for_class(a__  ,  a__)
       SCREAMING_SNAKE_CASE				    =		inputs['''input_ids'''].shape[0]
       with torch.no_grad():
        SCREAMING_SNAKE_CASE				    =		model(**a__)
       SCREAMING_SNAKE_CASE				    =		outputs.attentions
       self.assertEqual(len(a__)  ,  self.model_tester.num_hidden_layers)

       # check that output_attentions also work using config
       del inputs_dict["output_attentions"]
       SCREAMING_SNAKE_CASE				    =		True
       SCREAMING_SNAKE_CASE				    =		model_class(a__)
       model.to(a__)
       model.eval()

       SCREAMING_SNAKE_CASE				    =		self._prepare_for_class(a__  ,  a__)
       SCREAMING_SNAKE_CASE				    =		inputs['''input_ids'''].shape[0]
       with torch.no_grad():
        SCREAMING_SNAKE_CASE				    =		model(**a__)
       SCREAMING_SNAKE_CASE				    =		outputs.attentions
       self.assertEqual(len(a__)  ,  self.model_tester.num_hidden_layers)

       self.assertListEqual(
           list(attentions[0].shape[-3:])  ,  [batch_size, seq_len, config.hidden_size]  ,  )
       SCREAMING_SNAKE_CASE				    =		len(a__)

       # Check attention is always last and order is fine
       SCREAMING_SNAKE_CASE				    =		True
       SCREAMING_SNAKE_CASE				    =		True
       SCREAMING_SNAKE_CASE				    =		model_class(a__)
       model.to(a__)
       model.eval()

       SCREAMING_SNAKE_CASE				    =		self._prepare_for_class(a__  ,  a__)
       SCREAMING_SNAKE_CASE				    =		inputs['''input_ids'''].shape[0]
       with torch.no_grad():
        SCREAMING_SNAKE_CASE				    =		model(**a__)

       SCREAMING_SNAKE_CASE				    =		1
       self.assertEqual(out_len + added_hidden_states  ,  len(a__))

       SCREAMING_SNAKE_CASE				    =		outputs.attentions

       self.assertEqual(len(a__)  ,  self.model_tester.num_hidden_layers)
       self.assertListEqual(
           list(self_attentions[0].shape[-3:])  ,  [batch_size, seq_len, config.hidden_size]  ,  )





     @slow
     def     __UpperCamelCase						(		self					:	Union[str, Any])							->	Dict:

      """simple docstring"""
      for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       SCREAMING_SNAKE_CASE				    =		RwkvModel.from_pretrained(a__)
       self.assertIsNotNone(a__)

@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0			,							reason='''See https://github.com/huggingface/transformers/pull/24204'''	)
@slow
class 	a_(		unittest.TestCase	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Union[str, Any])							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		'''RWKV/rwkv-4-169m-pile'''
      SCREAMING_SNAKE_CASE				    =		AutoTokenizer.from_pretrained(self.model_id)





     def     __UpperCamelCase						(		self					:	Optional[Any])							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		'''Hello my name is Jasmine and I am a newbie to the'''
      SCREAMING_SNAKE_CASE				    =		RwkvForCausalLM.from_pretrained(self.model_id).to(a__)

      SCREAMING_SNAKE_CASE				    =		self.tokenizer('Hello my name is'  ,  return_tensors='pt').input_ids.to(a__)
      SCREAMING_SNAKE_CASE				    =		model.generate(a__  ,  max_new_tokens=1_0)
      SCREAMING_SNAKE_CASE				    =		self.tokenizer.decode(output[0].tolist())

      self.assertEqual(a__  ,  a__)





     def     __UpperCamelCase						(		self					:	List[Any])							->	Optional[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		'''Hello my name is Jasmine and I am a newbie to the'''

      SCREAMING_SNAKE_CASE				    =		self.tokenizer('Hello my name is'  ,  return_tensors='pt').input_ids.to(a__)
      SCREAMING_SNAKE_CASE				    =		RwkvForCausalLM.from_pretrained(self.model_id  ,  torch_dtype=torch.bfloataa).to(a__)

      SCREAMING_SNAKE_CASE				    =		model.generate(a__  ,  max_new_tokens=1_0)
      SCREAMING_SNAKE_CASE				    =		self.tokenizer.decode(output[0].tolist())

      self.assertEqual(a__  ,  a__)










'''simple docstring'''





import math
from decimal import Decimal, getcontext


def 					UpperCamelCase						(	snake_case = 4_00_00_00 ):




   '''simple docstring'''

   try:
      SCREAMING_SNAKE_CASE__      :					List[Any]							    =			int(snake_case )
   except (TypeError, ValueError):
      raise TypeError('''Parameter n must be int or castable to int.''' )
   if n <= 0:
      raise ValueError('''Parameter n must be greater than or equal to one.''' )
   SCREAMING_SNAKE_CASE__      :					Optional[int]							    =			1_00
   SCREAMING_SNAKE_CASE__      :					int							    =			(Decimal(5 ) ** Decimal(0.5 ) + 1) / Decimal(2 )

   SCREAMING_SNAKE_CASE__      :					int							    =			(math.floor(math.log(n * (phi + 2)						,  snake_case ) - 1 ) // 3) * 3 + 2
   SCREAMING_SNAKE_CASE__      :					int							    =			Decimal(round(phi ** Decimal(index + 1 ) ) ) / (phi + 2)
   SCREAMING_SNAKE_CASE__      :					Any							    =			num // 2
   return int(snake_case )


if __name__ == "__main__":
       print(F"{solution() = }")

