'''simple docstring'''




import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
      RwkvForCausalLM,
      RwkvModel,
  )
  from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
  __snake_case  : int		       =    False




class       __UpperCAmelCase							:





       '''simple docstring'''



       def __init__( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=14       ,	_SCREAMING_SNAKE_CASE=7       ,	_SCREAMING_SNAKE_CASE=True       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=True       ,	_SCREAMING_SNAKE_CASE=True       ,	_SCREAMING_SNAKE_CASE=True       ,	_SCREAMING_SNAKE_CASE=99       ,	_SCREAMING_SNAKE_CASE=32       ,	_SCREAMING_SNAKE_CASE=5       ,	_SCREAMING_SNAKE_CASE=37       ,	_SCREAMING_SNAKE_CASE="gelu"       ,	_SCREAMING_SNAKE_CASE=0.1       ,	_SCREAMING_SNAKE_CASE=0.1       ,	_SCREAMING_SNAKE_CASE=512       ,	_SCREAMING_SNAKE_CASE=16       ,	_SCREAMING_SNAKE_CASE=2       ,	_SCREAMING_SNAKE_CASE=3       ,	_SCREAMING_SNAKE_CASE=4       ,	_SCREAMING_SNAKE_CASE=None       ,	)						->	int:
           A_      =  parent
           A_      =  batch_size
           A_      =  seq_length
           A_      =  is_training
           A_      =  use_token_type_ids
           A_      =  use_input_mask
           A_      =  use_labels
           A_      =  use_mc_token_ids
           A_      =  vocab_size
           A_      =  hidden_size
           A_      =  num_hidden_layers
           A_      =  intermediate_size
           A_      =  hidden_act
           A_      =  hidden_dropout_prob
           A_      =  attention_probs_dropout_prob
           A_      =  max_position_embeddings
           A_      =  type_vocab_size
           A_      =  type_sequence_label_size
           A_      =  num_labels
           A_      =  num_choices
           A_      =  scope
           A_      =  vocab_size - 1
           A_      =  vocab_size - 1
           A_      =  vocab_size - 1

       def    __A			( self					)						->	List[str]:
           return RwkvConfig.from_pretrained('''sgugger/rwkv-4-pile-7b'''					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=False					)						->	Optional[int]:
           A_      =  ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size					)

           A_      =  None
           if self.use_input_mask:
               A_      =  random_attention_mask([self.batch_size, self.seq_length]					)

           A_      =  None
           if self.use_token_type_ids:
               A_      =  ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size					)

           A_      =  None
           if self.use_mc_token_ids:
               A_      =  ids_tensor([self.batch_size, self.num_choices]       ,	self.seq_length					)

           A_      =  None
           A_      =  None
           A_      =  None
           if self.use_labels:
               A_      =  ids_tensor([self.batch_size]       ,	self.type_sequence_label_size					)
               A_      =  ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels					)
               A_      =  ids_tensor([self.batch_size]       ,	self.num_choices					)

           A_      =  self.get_config(
               gradient_checkpointing=_SCREAMING_SNAKE_CASE       ,	scale_attn_by_inverse_layer_idx=_SCREAMING_SNAKE_CASE       ,	reorder_and_upcast_attn=_SCREAMING_SNAKE_CASE       ,	)

           return (
               config,
               input_ids,
               input_mask,
               None,
               token_type_ids,
               mc_token_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=False					)						->	Tuple:
           return RwkvConfig(
               vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	intermediate_size=self.intermediate_size       ,	activation_function=self.hidden_act       ,	resid_pdrop=self.hidden_dropout_prob       ,	attn_pdrop=self.attention_probs_dropout_prob       ,	n_positions=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	use_cache=_SCREAMING_SNAKE_CASE       ,	bos_token_id=self.bos_token_id       ,	eos_token_id=self.eos_token_id       ,	pad_token_id=self.pad_token_id       ,	gradient_checkpointing=_SCREAMING_SNAKE_CASE       ,	scale_attn_by_inverse_layer_idx=_SCREAMING_SNAKE_CASE       ,	reorder_and_upcast_attn=_SCREAMING_SNAKE_CASE       ,	)

       def    __A			( self					)						->	int:
           A_      =  self.get_config()
           A_      =  300
           return config

       def    __A			( self					)						->	List[Any]:
           (
               (
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,
           )      =  self.prepare_config_and_inputs()

           A_      =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]					)
           A_      =  ids_tensor([self.batch_size, self.seq_length]       ,	vocab_size=2					)

           return (
               config,
               input_ids,
               input_mask,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	*_SCREAMING_SNAKE_CASE					)						->	int:
           A_      =  True
           A_      =  RwkvModel(config=_SCREAMING_SNAKE_CASE					)
           model.to(_SCREAMING_SNAKE_CASE					)
           model.eval()

           A_      =  model(_SCREAMING_SNAKE_CASE					)

           self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)					)
           self.parent.assertEqual(len(result.hidden_states					)       ,	config.num_hidden_layers + 1					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	*_SCREAMING_SNAKE_CASE					)						->	Tuple:
           A_      =  RwkvForCausalLM(_SCREAMING_SNAKE_CASE					)
           model.to(_SCREAMING_SNAKE_CASE					)
           model.eval()

           A_      =  model(_SCREAMING_SNAKE_CASE       ,	labels=_SCREAMING_SNAKE_CASE					)
           self.parent.assertEqual(result.loss.shape       ,	()					)
           self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	*_SCREAMING_SNAKE_CASE					)						->	str:
           A_      =  RwkvModel(config=_SCREAMING_SNAKE_CASE					)
           model.to(_SCREAMING_SNAKE_CASE					)
           model.eval()

           A_      =  model(_SCREAMING_SNAKE_CASE					)
           A_      =  outputs.last_hidden_state

           A_      =  model(input_ids[:, :2]					)
           A_      =  outputs.last_hidden_state

           # Using the state computed on the first inputs, we will get the same output
           A_      =  model(input_ids[:, 2:]       ,	state=outputs.state					)
           A_      =  outputs.last_hidden_state

           self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]       ,	dim=1					)       ,	_SCREAMING_SNAKE_CASE       ,	atol=1E-5					)					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	*_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=False					)						->	Dict:
           A_      =  RwkvForCausalLM(_SCREAMING_SNAKE_CASE					)
           model.to(_SCREAMING_SNAKE_CASE					)
           if gradient_checkpointing:
               model.gradient_checkpointing_enable()

           A_      =  model(_SCREAMING_SNAKE_CASE       ,	labels=_SCREAMING_SNAKE_CASE					)
           self.parent.assertEqual(result.loss.shape       ,	()					)
           self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)					)
           result.loss.backward()

       def    __A			( self					)						->	List[str]:
           A_      =  self.prepare_config_and_inputs()

           (
               (
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,(
               A_
           )  ,
           )      =  config_and_inputs

           A_      =  {'''input_ids''': input_ids}

           return config, inputs_dict




@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0		, reason='See https://github.com/huggingface/transformers/pull/24204'	)
@require_torch
class       __UpperCAmelCase							(							_UpperCamelCase		, _UpperCamelCase		, _UpperCamelCase		, unittest.TestCase	):





       '''simple docstring'''



       __lowercase   :   str  					=							(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
       __lowercase   :   Tuple  					=							(
           {'feature-extraction': RwkvModel, 'text-generation': RwkvForCausalLM} if is_torch_available() else {}
       )
       # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
       __lowercase   :   Dict  					=							False
       __lowercase   :   Tuple  					=							False
       __lowercase   :   Tuple  					=							False
       __lowercase   :   Union[str, Any]  					=							False
       __lowercase   :   Union[str, Any]  					=							False  # Rwkv does not support head masking

       def    __A			( self					)						->	str:
           A_      =  RwkvModelTester(self					)
           A_      =  ConfigTester(
               self       ,	config_class=_SCREAMING_SNAKE_CASE       ,	n_embd=37       ,	common_properties=['''hidden_size''', '''num_hidden_layers''']					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None					)						->	str:
           if isinstance(_SCREAMING_SNAKE_CASE       ,	torch.Tensor					):
               A_  ,A_      =  member.max().item(), member.min().item()
           elif isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					) or isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					):
               A_  ,A_      =  max(_SCREAMING_SNAKE_CASE					), min(_SCREAMING_SNAKE_CASE					)

           if not isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					):
               raise TypeError('''container should be a list or tuple'''					)
           elif len(_SCREAMING_SNAKE_CASE					) != 2:
               raise ValueError('''container should have 2 elements'''					)

           A_  ,A_      =  container

           A_      =  (min_value >= expected_min) and (max_value <= expected_max)

           if not is_inside_interval:
               A_      =  '''%s not found in %s''' % (safe_repr(_SCREAMING_SNAKE_CASE					), safe_repr(_SCREAMING_SNAKE_CASE					))
               self.fail(self._formatMessage(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)					)

       def    __A			( self					)						->	Union[str, Any]:
           self.config_tester.run_common_tests()

       def    __A			( self					)						->	int:
           A_      =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_rwkv_model(*_SCREAMING_SNAKE_CASE					)

       def    __A			( self					)						->	Union[str, Any]:
           A_      =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_causl_lm(*_SCREAMING_SNAKE_CASE					)

       def    __A			( self					)						->	List[str]:
           A_      =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_state_equivalency(*_SCREAMING_SNAKE_CASE					)

       def    __A			( self					)						->	str:
           A_  ,A_      =  self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               A_      =  model_class(config=_SCREAMING_SNAKE_CASE					)
               for name, param in model.named_parameters():
                   if "time_decay" in name:
                       if param.requires_grad:
                           self.assertTrue(param.data.max().item() == 3.0					)
                           self.assertTrue(param.data.min().item() == -5.0					)
                   elif "time_first" in name:
                       if param.requires_grad:
                           # check if it's a ones like
                           self.assertTrue(torch.allclose(param.data       ,	torch.ones_like(param.data					)       ,	atol=1E-5       ,	rtol=1E-5					)					)
                   elif any(x in name for x in ['''time_mix_key''', '''time_mix_receptance''']					):
                       if param.requires_grad:
                           self.assertInterval(
                               param.data       ,	[0.0, 1.0]       ,	msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''       ,	)
                   elif "time_mix_value" in name:
                       if param.requires_grad:
                           self.assertInterval(
                               param.data       ,	[0.0, 1.3]       ,	msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''       ,	)

       def    __A			( self					)						->	str:
           A_  ,A_      =  self.model_tester.prepare_config_and_inputs_for_common()
           A_      =  True

           A_      =  getattr(self.model_tester       ,	'''seq_length'''       ,	_SCREAMING_SNAKE_CASE					)

           for model_class in self.all_model_classes:
               A_      =  True
               A_      =  False
               A_      =  True
               A_      =  model_class(_SCREAMING_SNAKE_CASE					)
               model.to(_SCREAMING_SNAKE_CASE					)
               model.eval()

               A_      =  self._prepare_for_class(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  inputs['''input_ids'''].shape[0]
               with torch.no_grad():
                   A_      =  model(**_SCREAMING_SNAKE_CASE					)
               A_      =  outputs.attentions
               self.assertEqual(len(_SCREAMING_SNAKE_CASE					)       ,	self.model_tester.num_hidden_layers					)

               # check that output_attentions also work using config
               del inputs_dict["output_attentions"]
               A_      =  True
               A_      =  model_class(_SCREAMING_SNAKE_CASE					)
               model.to(_SCREAMING_SNAKE_CASE					)
               model.eval()

               A_      =  self._prepare_for_class(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  inputs['''input_ids'''].shape[0]
               with torch.no_grad():
                   A_      =  model(**_SCREAMING_SNAKE_CASE					)
               A_      =  outputs.attentions
               self.assertEqual(len(_SCREAMING_SNAKE_CASE					)       ,	self.model_tester.num_hidden_layers					)

               self.assertListEqual(
                   list(attentions[0].shape[-3:]					)       ,	[batch_size, seq_len, config.hidden_size]       ,	)
               A_      =  len(_SCREAMING_SNAKE_CASE					)

               # Check attention is always last and order is fine
               A_      =  True
               A_      =  True
               A_      =  model_class(_SCREAMING_SNAKE_CASE					)
               model.to(_SCREAMING_SNAKE_CASE					)
               model.eval()

               A_      =  self._prepare_for_class(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  inputs['''input_ids'''].shape[0]
               with torch.no_grad():
                   A_      =  model(**_SCREAMING_SNAKE_CASE					)

               A_      =  1
               self.assertEqual(out_len + added_hidden_states       ,	len(_SCREAMING_SNAKE_CASE					)					)

               A_      =  outputs.attentions

               self.assertEqual(len(_SCREAMING_SNAKE_CASE					)       ,	self.model_tester.num_hidden_layers					)
               self.assertListEqual(
                   list(self_attentions[0].shape[-3:]					)       ,	[batch_size, seq_len, config.hidden_size]       ,	)

       @slow
       def    __A			( self					)						->	Optional[Any]:
           for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               A_      =  RwkvModel.from_pretrained(_SCREAMING_SNAKE_CASE					)
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)




@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0		, reason='See https://github.com/huggingface/transformers/pull/24204'	)
@slow
class       __UpperCAmelCase							(							unittest.TestCase	):





       '''simple docstring'''



       def    __A			( self					)						->	Any:
           A_      =  '''RWKV/rwkv-4-169m-pile'''
           A_      =  AutoTokenizer.from_pretrained(self.model_id					)

       def    __A			( self					)						->	Any:
           A_      =  '''Hello my name is Jasmine and I am a newbie to the'''
           A_      =  RwkvForCausalLM.from_pretrained(self.model_id					).to(_SCREAMING_SNAKE_CASE					)

           A_      =  self.tokenizer('''Hello my name is'''       ,	return_tensors='''pt'''					).input_ids.to(_SCREAMING_SNAKE_CASE					)
           A_      =  model.generate(_SCREAMING_SNAKE_CASE       ,	max_new_tokens=10					)
           A_      =  self.tokenizer.decode(output[0].tolist()					)

           self.assertEqual(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)

       def    __A			( self					)						->	Tuple:
           A_      =  '''Hello my name is Jasmine and I am a newbie to the'''

           A_      =  self.tokenizer('''Hello my name is'''       ,	return_tensors='''pt'''					).input_ids.to(_SCREAMING_SNAKE_CASE					)
           A_      =  RwkvForCausalLM.from_pretrained(self.model_id       ,	torch_dtype=torch.bfloataa					).to(_SCREAMING_SNAKE_CASE					)

           A_      =  model.generate(_SCREAMING_SNAKE_CASE       ,	max_new_tokens=10					)
           A_      =  self.tokenizer.decode(output[0].tolist()					)

           self.assertEqual(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)







'''simple docstring'''




__snake_case  : str		       =    10**8





def 	_UpperCAmelCase       (				_UpperCamelCase       :      int  )      -> bool:
    if n % 10 == 0:
        return False
    A_      =  str(_UpperCamelCase  )
    return s == s[::-1]





def 	_UpperCAmelCase       (				)      -> int:
    A_      =  set()
    A_      =  1
    A_      =  5
    while sum_squares < LIMIT:
        A_      =  first_square + 1
        while sum_squares < LIMIT:
            if is_palindrome(_UpperCamelCase  ):
                answer.add(_UpperCamelCase  )
            last_square += 1
            sum_squares += last_square**2
        first_square += 1
        A_      =  first_square**2 + (first_square + 1) ** 2

    return sum(_UpperCamelCase  )


if __name__ == "__main__":
  print(solution())






