






"""simple docstring"""



import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
     RwkvForCausalLM,
     RwkvModel,
 )
 from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
 a  :						Union[str, Any]					       =							False


class 	_UpperCamelCase :



    '''simple docstring'''


    def __init__(      self   ,			__lowercase   ,			__lowercase=14   ,			__lowercase=7   ,			__lowercase=True   ,			__lowercase=False   ,			__lowercase=True   ,			__lowercase=True   ,			__lowercase=True   ,			__lowercase=99   ,			__lowercase=32   ,			__lowercase=5   ,			__lowercase=37   ,			__lowercase="gelu"   ,			__lowercase=0.1   ,			__lowercase=0.1   ,			__lowercase=512   ,			__lowercase=16   ,			__lowercase=2   ,			__lowercase=3   ,			__lowercase=4   ,			__lowercase=None   ,			):
     UpperCAmelCase__ 						=				parent
     UpperCAmelCase__ 						=				batch_size
     UpperCAmelCase__ 						=				seq_length
     UpperCAmelCase__ 						=				is_training
     UpperCAmelCase__ 						=				use_token_type_ids
     UpperCAmelCase__ 						=				use_input_mask
     UpperCAmelCase__ 						=				use_labels
     UpperCAmelCase__ 						=				use_mc_token_ids
     UpperCAmelCase__ 						=				vocab_size
     UpperCAmelCase__ 						=				hidden_size
     UpperCAmelCase__ 						=				num_hidden_layers
     UpperCAmelCase__ 						=				intermediate_size
     UpperCAmelCase__ 						=				hidden_act
     UpperCAmelCase__ 						=				hidden_dropout_prob
     UpperCAmelCase__ 						=				attention_probs_dropout_prob
     UpperCAmelCase__ 						=				max_position_embeddings
     UpperCAmelCase__ 						=				type_vocab_size
     UpperCAmelCase__ 						=				type_sequence_label_size
     UpperCAmelCase__ 						=				num_labels
     UpperCAmelCase__ 						=				num_choices
     UpperCAmelCase__ 						=				scope
     UpperCAmelCase__ 						=				vocab_size - 1
     UpperCAmelCase__ 						=				vocab_size - 1
     UpperCAmelCase__ 						=				vocab_size - 1


    def 		A__      (      self			):
     return RwkvConfig.from_pretrained("""sgugger/rwkv-4-pile-7b"""			)


    def 		A__      (      self   ,			__lowercase=False   ,			__lowercase=False   ,			__lowercase=False			):
     UpperCAmelCase__ 						=				ids_tensor([self.batch_size, self.seq_length]   ,			self.vocab_size			)

     UpperCAmelCase__ 						=				None
     if self.use_input_mask:
      UpperCAmelCase__ 						=				random_attention_mask([self.batch_size, self.seq_length]			)

     UpperCAmelCase__ 						=				None
     if self.use_token_type_ids:
      UpperCAmelCase__ 						=				ids_tensor([self.batch_size, self.seq_length]   ,			self.type_vocab_size			)

     UpperCAmelCase__ 						=				None
     if self.use_mc_token_ids:
      UpperCAmelCase__ 						=				ids_tensor([self.batch_size, self.num_choices]   ,			self.seq_length			)

     UpperCAmelCase__ 						=				None
     UpperCAmelCase__ 						=				None
     UpperCAmelCase__ 						=				None
     if self.use_labels:
      UpperCAmelCase__ 						=				ids_tensor([self.batch_size]   ,			self.type_sequence_label_size			)
      UpperCAmelCase__ 						=				ids_tensor([self.batch_size, self.seq_length]   ,			self.num_labels			)
      UpperCAmelCase__ 						=				ids_tensor([self.batch_size]   ,			self.num_choices			)

     UpperCAmelCase__ 						=				self.get_config(
         gradient_checkpointing=__lowercase   ,			scale_attn_by_inverse_layer_idx=__lowercase   ,			reorder_and_upcast_attn=__lowercase   ,			)

     return (
         config,
         input_ids,
         input_mask,
         None,
         token_type_ids,
         mc_token_ids,
         sequence_labels,
         token_labels,
         choice_labels,
     )


    def 		A__      (      self   ,			__lowercase=False   ,			__lowercase=False   ,			__lowercase=False			):
     return RwkvConfig(
         vocab_size=self.vocab_size   ,			hidden_size=self.hidden_size   ,			num_hidden_layers=self.num_hidden_layers   ,			intermediate_size=self.intermediate_size   ,			activation_function=self.hidden_act   ,			resid_pdrop=self.hidden_dropout_prob   ,			attn_pdrop=self.attention_probs_dropout_prob   ,			n_positions=self.max_position_embeddings   ,			type_vocab_size=self.type_vocab_size   ,			use_cache=__lowercase   ,			bos_token_id=self.bos_token_id   ,			eos_token_id=self.eos_token_id   ,			pad_token_id=self.pad_token_id   ,			gradient_checkpointing=__lowercase   ,			scale_attn_by_inverse_layer_idx=__lowercase   ,			reorder_and_upcast_attn=__lowercase   ,			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.get_config()
     UpperCAmelCase__ 						=				300
     return config


    def 		A__      (      self			):
     (
         (
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	
     ) 						=				self.prepare_config_and_inputs()

     UpperCAmelCase__ 						=				floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
     UpperCAmelCase__ 						=				ids_tensor([self.batch_size, self.seq_length]   ,			vocab_size=2			)

     return (
         config,
         input_ids,
         input_mask,
         head_mask,
         token_type_ids,
         sequence_labels,
         token_labels,
         choice_labels,
         encoder_hidden_states,
         encoder_attention_mask,
     )


    def 		A__      (      self   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			*__lowercase			):
     UpperCAmelCase__ 						=				True
     UpperCAmelCase__ 						=				RwkvModel(config=__lowercase			)
     model.to(__lowercase			)
     model.eval()

     UpperCAmelCase__ 						=				model(__lowercase			)

     self.parent.assertEqual(result.last_hidden_state.shape   ,			(self.batch_size, self.seq_length, self.hidden_size)			)
     self.parent.assertEqual(len(result.hidden_states			)   ,			config.num_hidden_layers + 1			)


    def 		A__      (      self   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			*__lowercase			):
     UpperCAmelCase__ 						=				RwkvForCausalLM(__lowercase			)
     model.to(__lowercase			)
     model.eval()

     UpperCAmelCase__ 						=				model(__lowercase   ,			labels=__lowercase			)
     self.parent.assertEqual(result.loss.shape   ,			()			)
     self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.seq_length, self.vocab_size)			)


    def 		A__      (      self   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			*__lowercase			):
     UpperCAmelCase__ 						=				RwkvModel(config=__lowercase			)
     model.to(__lowercase			)
     model.eval()

     UpperCAmelCase__ 						=				model(__lowercase			)
     UpperCAmelCase__ 						=				outputs.last_hidden_state

     UpperCAmelCase__ 						=				model(input_ids[:, :2]			)
     UpperCAmelCase__ 						=				outputs.last_hidden_state

     # Using the state computed on the first inputs, we will get the same output
     UpperCAmelCase__ 						=				model(input_ids[:, 2:]   ,			state=outputs.state			)
     UpperCAmelCase__ 						=				outputs.last_hidden_state

     self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]   ,			dim=1			)   ,			__lowercase   ,			atol=1e-5			)			)


    def 		A__      (      self   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			__lowercase   ,			*__lowercase   ,			__lowercase=False			):
     UpperCAmelCase__ 						=				RwkvForCausalLM(__lowercase			)
     model.to(__lowercase			)
     if gradient_checkpointing:
      model.gradient_checkpointing_enable()

     UpperCAmelCase__ 						=				model(__lowercase   ,			labels=__lowercase			)
     self.parent.assertEqual(result.loss.shape   ,			()			)
     self.parent.assertEqual(result.logits.shape   ,			(self.batch_size, self.seq_length, self.vocab_size)			)
     result.loss.backward()






    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.prepare_config_and_inputs()

     (
         (
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	(
         UpperCAmelCase__
     )						,	
     ) 						=				config_and_inputs

     UpperCAmelCase__ 						=				{"""input_ids""": input_ids}

     return config, inputs_dict



@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0		,     reason='See https://github.com/huggingface/transformers/pull/24204'							)
@require_torch
class 	_UpperCamelCase ( __UpperCamelCase		,     __UpperCamelCase		,     __UpperCamelCase		,     unittest.TestCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
    __lowercase    :   Tuple											=						(
        {'feature-extraction': RwkvModel, 'text-generation': RwkvForCausalLM} if is_torch_available() else {}
    )
    # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
    __lowercase    :   List[str]											=						False
    __lowercase    :   Tuple											=						False
    __lowercase    :   Any											=						False
    __lowercase    :   Dict											=						False
    __lowercase    :   str											=						False  # Rwkv does not support head masking


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				RwkvModelTester(self			)
     UpperCAmelCase__ 						=				ConfigTester(
         self   ,			config_class=__lowercase   ,			n_embd=37   ,			common_properties=["""hidden_size""", """num_hidden_layers"""]			)


    def 		A__      (      self   ,			__lowercase   ,			__lowercase   ,			__lowercase=None			):
     if isinstance(__lowercase   ,			torch.Tensor			):
      UpperCAmelCase__						,	UpperCAmelCase__ 						=				member.max().item(), member.min().item()
     elif isinstance(__lowercase   ,			__lowercase			) or isinstance(__lowercase   ,			__lowercase			):
      UpperCAmelCase__						,	UpperCAmelCase__ 						=				max(__lowercase			), min(__lowercase			)

     if not isinstance(__lowercase   ,			__lowercase			):
      raise TypeError("""container should be a list or tuple"""			)
     elif len(__lowercase			) != 2:
      raise ValueError("""container should have 2 elements"""			)

     UpperCAmelCase__						,	UpperCAmelCase__ 						=				container

     UpperCAmelCase__ 						=				(min_value >= expected_min) and (max_value <= expected_max)

     if not is_inside_interval:
      UpperCAmelCase__ 						=				"""%s not found in %s""" % (safe_repr(__lowercase			), safe_repr(__lowercase			))
      self.fail(self._formatMessage(__lowercase   ,			__lowercase			)			)


    def 		A__      (      self			):
     self.config_tester.run_common_tests()


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_rwkv_model(*__lowercase			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_causl_lm(*__lowercase			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_state_equivalency(*__lowercase			)


    def 		A__      (      self			):
     UpperCAmelCase__						,	UpperCAmelCase__ 						=				self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      UpperCAmelCase__ 						=				model_class(config=__lowercase			)
      for name, param in model.named_parameters():
       if "time_decay" in name:
        if param.requires_grad:
         self.assertTrue(param.data.max().item() == 3.0			)
         self.assertTrue(param.data.min().item() == -5.0			)
       elif "time_first" in name:
        if param.requires_grad:
         # check if it's a ones like
         self.assertTrue(torch.allclose(param.data   ,			torch.ones_like(param.data			)   ,			atol=1e-5   ,			rtol=1e-5			)			)
       elif any(x in name for x in ["""time_mix_key""", """time_mix_receptance"""]			):
        if param.requires_grad:
         self.assertInterval(
             param.data   ,			[0.0, 1.0]   ,			msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''   ,			)
       elif "time_mix_value" in name:
        if param.requires_grad:
         self.assertInterval(
             param.data   ,			[0.0, 1.3]   ,			msg=F'''Parameter {name} of model {model_class} seems not properly initialized'''   ,			)


    def 		A__      (      self			):
     UpperCAmelCase__						,	UpperCAmelCase__ 						=				self.model_tester.prepare_config_and_inputs_for_common()
     UpperCAmelCase__ 						=				True

     UpperCAmelCase__ 						=				getattr(self.model_tester   ,			"""seq_length"""   ,			__lowercase			)

     for model_class in self.all_model_classes:
      UpperCAmelCase__ 						=				True
      UpperCAmelCase__ 						=				False
      UpperCAmelCase__ 						=				True
      UpperCAmelCase__ 						=				model_class(__lowercase			)
      model.to(__lowercase			)
      model.eval()

      UpperCAmelCase__ 						=				self._prepare_for_class(__lowercase   ,			__lowercase			)
      UpperCAmelCase__ 						=				inputs["""input_ids"""].shape[0]
      with torch.no_grad():
       UpperCAmelCase__ 						=				model(**__lowercase			)
      UpperCAmelCase__ 						=				outputs.attentions
      self.assertEqual(len(__lowercase			)   ,			self.model_tester.num_hidden_layers			)

      # check that output_attentions also work using config
      del inputs_dict["output_attentions"]
      UpperCAmelCase__ 						=				True
      UpperCAmelCase__ 						=				model_class(__lowercase			)
      model.to(__lowercase			)
      model.eval()

      UpperCAmelCase__ 						=				self._prepare_for_class(__lowercase   ,			__lowercase			)
      UpperCAmelCase__ 						=				inputs["""input_ids"""].shape[0]
      with torch.no_grad():
       UpperCAmelCase__ 						=				model(**__lowercase			)
      UpperCAmelCase__ 						=				outputs.attentions
      self.assertEqual(len(__lowercase			)   ,			self.model_tester.num_hidden_layers			)

      self.assertListEqual(
          list(attentions[0].shape[-3:]			)   ,			[batch_size, seq_len, config.hidden_size]   ,			)
      UpperCAmelCase__ 						=				len(__lowercase			)

      # Check attention is always last and order is fine
      UpperCAmelCase__ 						=				True
      UpperCAmelCase__ 						=				True
      UpperCAmelCase__ 						=				model_class(__lowercase			)
      model.to(__lowercase			)
      model.eval()

      UpperCAmelCase__ 						=				self._prepare_for_class(__lowercase   ,			__lowercase			)
      UpperCAmelCase__ 						=				inputs["""input_ids"""].shape[0]
      with torch.no_grad():
       UpperCAmelCase__ 						=				model(**__lowercase			)

      UpperCAmelCase__ 						=				1
      self.assertEqual(out_len + added_hidden_states   ,			len(__lowercase			)			)

      UpperCAmelCase__ 						=				outputs.attentions

      self.assertEqual(len(__lowercase			)   ,			self.model_tester.num_hidden_layers			)
      self.assertListEqual(
          list(self_attentions[0].shape[-3:]			)   ,			[batch_size, seq_len, config.hidden_size]   ,			)






    @slow
    def 		A__      (      self			):
     for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
      UpperCAmelCase__ 						=				RwkvModel.from_pretrained(__lowercase			)
      self.assertIsNotNone(__lowercase			)



@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0		,     reason='See https://github.com/huggingface/transformers/pull/24204'							)
@slow
class 	_UpperCamelCase ( unittest.TestCase							):



    '''simple docstring'''


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				"""RWKV/rwkv-4-169m-pile"""
     UpperCAmelCase__ 						=				AutoTokenizer.from_pretrained(self.model_id			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				"""Hello my name is Jasmine and I am a newbie to the"""
     UpperCAmelCase__ 						=				RwkvForCausalLM.from_pretrained(self.model_id			).to(__lowercase			)

     UpperCAmelCase__ 						=				self.tokenizer("""Hello my name is"""   ,			return_tensors="""pt"""			).input_ids.to(__lowercase			)
     UpperCAmelCase__ 						=				model.generate(__lowercase   ,			max_new_tokens=10			)
     UpperCAmelCase__ 						=				self.tokenizer.decode(output[0].tolist()			)

     self.assertEqual(__lowercase   ,			__lowercase			)






    def 		A__      (      self			):
     UpperCAmelCase__ 						=				"""Hello my name is Jasmine and I am a newbie to the"""

     UpperCAmelCase__ 						=				self.tokenizer("""Hello my name is"""   ,			return_tensors="""pt"""			).input_ids.to(__lowercase			)
     UpperCAmelCase__ 						=				RwkvForCausalLM.from_pretrained(self.model_id   ,			torch_dtype=torch.bfloataa			).to(__lowercase			)

     UpperCAmelCase__ 						=				model.generate(__lowercase   ,			max_new_tokens=10			)
     UpperCAmelCase__ 						=				self.tokenizer.decode(output[0].tolist()			)

     self.assertEqual(__lowercase   ,			__lowercase			)











"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


a  :						List[Any]					       =							logging.get_logger(__name__)

a  :						List[Any]					       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a  :						Any					       =							{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

a  :						str					       =							{
    '''microsoft/mpnet-base''': 512,
}

a  :						Union[str, Any]					       =							{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}


class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						VOCAB_FILES_NAMES
    __lowercase    :   Optional[int]											=						PRETRAINED_VOCAB_FILES_MAP
    __lowercase    :   Dict											=						PRETRAINED_INIT_CONFIGURATION
    __lowercase    :   Optional[Any]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowercase    :   Dict											=						MPNetTokenizer
    __lowercase    :   Dict											=						['input_ids', 'attention_mask']


    def __init__(      self   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=True   ,			__lowercase="<s>"   ,			__lowercase="</s>"   ,			__lowercase="</s>"   ,			__lowercase="<s>"   ,			__lowercase="[UNK]"   ,			__lowercase="<pad>"   ,			__lowercase="<mask>"   ,			__lowercase=True   ,			__lowercase=None   ,			**__lowercase   ,			):
     super().__init__(
         __lowercase   ,			tokenizer_file=__lowercase   ,			do_lower_case=__lowercase   ,			bos_token=__lowercase   ,			eos_token=__lowercase   ,			sep_token=__lowercase   ,			cls_token=__lowercase   ,			unk_token=__lowercase   ,			pad_token=__lowercase   ,			mask_token=__lowercase   ,			tokenize_chinese_chars=__lowercase   ,			strip_accents=__lowercase   ,			**__lowercase   ,			)

     UpperCAmelCase__ 						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         pre_tok_state.get("""lowercase"""   ,			__lowercase			) != do_lower_case
         or pre_tok_state.get("""strip_accents"""   ,			__lowercase			) != strip_accents
     ):
      UpperCAmelCase__ 						=				getattr(__lowercase   ,			pre_tok_state.pop("""type"""			)			)
      UpperCAmelCase__ 						=				do_lower_case
      UpperCAmelCase__ 						=				strip_accents
      UpperCAmelCase__ 						=				pre_tok_class(**__lowercase			)

     UpperCAmelCase__ 						=				do_lower_case


    @property
    def 		A__      (      self			):
     if self._mask_token is None:
      if self.verbose:
       logger.error("""Using mask_token, but it is not set yet."""			)
      return None
     return str(self._mask_token			)


    @mask_token.setter
    def 		A__      (      self   ,			__lowercase			):
     UpperCAmelCase__ 						=				AddedToken(__lowercase   ,			lstrip=__lowercase   ,			rstrip=__lowercase			) if isinstance(__lowercase   ,			__lowercase			) else value
     UpperCAmelCase__ 						=				value


    def 		A__      (      self   ,			__lowercase   ,			__lowercase=None			):
     UpperCAmelCase__ 						=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
      return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				[self.sep_token_id]
     UpperCAmelCase__ 						=				[self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				self._tokenizer.model.save(__lowercase   ,			name=__lowercase			)
     return tuple(__lowercase			)



