





import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
          SpeechaTextaDecoder,
          SpeechaTextaForCausalLM,
      )







@require_torch
class        __snake_case			:





       '''simple docstring'''







       def __init__(    self					:       Optional[Any]				,							A					:       List[Any]				,							A					:       Tuple=99				,							A					:       Optional[Any]=13				,							A					:       List[str]=16				,							A					:       Tuple=7				,							A					:       List[str]=True				,							A					:       Union[str, Any]=True				,							A					:       Optional[int]=True				,							A					:       Dict=False				,							A					:       List[str]=True				,							A					:       List[str]=2				,							A					:       Any=32				,							A					:       List[str]=4				,							A					:       str=4				,							A					:       List[Any]=30				,							A					:       str=0				,							A					:       List[Any]=1				,							A					:       List[str]=2				,							A					:       Optional[Any]=None				,							):
         __snake_case:		Union[str, Any]					    =		parent
         __snake_case:		Optional[int]					    =		batch_size
         __snake_case:		List[str]					    =		decoder_seq_length
         # For common tests
         __snake_case:		List[str]					    =		self.decoder_seq_length
         __snake_case:		Union[str, Any]					    =		is_training
         __snake_case:		Optional[Any]					    =		use_attention_mask
         __snake_case:		List[Any]					    =		use_labels

         __snake_case:		Dict					    =		vocab_size
         __snake_case:		Optional[Any]					    =		d_model
         __snake_case:		str					    =		d_model
         __snake_case:		Optional[int]					    =		decoder_layers
         __snake_case:		List[str]					    =		decoder_layers
         __snake_case:		str					    =		decoder_ffn_dim
         __snake_case:		List[str]					    =		decoder_attention_heads
         __snake_case:		str					    =		decoder_attention_heads
         __snake_case:		Union[str, Any]					    =		eos_token_id
         __snake_case:		Union[str, Any]					    =		bos_token_id
         __snake_case:		List[Any]					    =		pad_token_id
         __snake_case:		Union[str, Any]					    =		decoder_start_token_id
         __snake_case:		List[str]					    =		use_cache
         __snake_case:		Any					    =		max_position_embeddings

         __snake_case:		List[str]					    =		None
         __snake_case:		List[str]					    =		decoder_seq_length
         __snake_case:		List[Any]					    =		2
         __snake_case:		List[str]					    =		1




       def 				UpperCAmelCase__					(    self					:       List[str]						):
         __snake_case:		Optional[Any]					    =		ids_tensor([self.batch_size, self.decoder_seq_length]				,							self.vocab_size						)

         __snake_case:		Union[str, Any]					    =		None
         if self.use_attention_mask:
           __snake_case:		str					    =		ids_tensor([self.batch_size, self.decoder_seq_length]				,							vocab_size=2						)

         __snake_case:		Optional[int]					    =		None
         if self.use_labels:
           __snake_case:		str					    =		ids_tensor([self.batch_size, self.decoder_seq_length]				,							self.vocab_size						)

         __snake_case:		Tuple					    =		SpeechaTextaConfig(
             vocab_size=self.vocab_size				,							d_model=self.d_model				,							decoder_layers=self.decoder_layers				,							decoder_ffn_dim=self.decoder_ffn_dim				,							decoder_attention_heads=self.decoder_attention_heads				,							eos_token_id=self.eos_token_id				,							bos_token_id=self.bos_token_id				,							use_cache=self.use_cache				,							pad_token_id=self.pad_token_id				,							decoder_start_token_id=self.decoder_start_token_id				,							max_position_embeddings=self.max_position_embeddings				,							)

         return (
             config,
             input_ids,
             attention_mask,
             lm_labels,
         )




       def 				UpperCAmelCase__					(    self					:       List[str]				,							A					:       List[str]				,							A					:       int				,							A					:       int				,							A					:       Dict				,							):
         __snake_case:		Union[str, Any]					    =		True
         __snake_case:		List[Any]					    =		SpeechaTextaDecoder(config=A						).to(A						).eval()
         __snake_case:		List[str]					    =		input_ids[:2]

         input_ids[input_ids == 0] += 1
         # first forward pass
         __snake_case:		str					    =		model(A				,							use_cache=A						)
         __snake_case:		Dict					    =		model(A						)
         __snake_case:		Any					    =		model(A				,							use_cache=A						)

         self.parent.assertTrue(len(A						) == len(A						)						)
         self.parent.assertTrue(len(A						) == len(A						) + 1						)

         __snake_case:		List[str]					    =		outputs["""past_key_values"""]

         # create hypothetical next token and extent to next_input_ids
         __snake_case:		str					    =		ids_tensor((2, 1)				,							config.vocab_size - 1						) + 1

         # append to next input_ids and
         __snake_case:		str					    =		torch.cat([input_ids, next_tokens]				,							dim=-1						)

         print(A						)

         __snake_case:		int					    =		model(A						)["""last_hidden_state"""]
         __snake_case:		Optional[Any]					    =		model(A				,							past_key_values=A						)["""last_hidden_state"""]

         # select random slice
         __snake_case:		Tuple					    =		ids_tensor((1,)				,							output_from_past.shape[-1]						).item()
         __snake_case:		Tuple					    =		output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
         __snake_case:		Tuple					    =		output_from_past[:, 0, random_slice_idx].detach()

         # test that outputs are equal for slice
         assert torch.allclose(A				,							A				,							atol=1E-3						)






       def 				UpperCAmelCase__					(    self					:       List[Any]						):
         __snake_case:		Optional[int]					    =		self.prepare_config_and_inputs()
         (
             (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  (
             __snake_case
         )    ,  
         ):		List[str]					    =		config_and_inputs

         __snake_case:		Optional[Any]					    =		{
             """input_ids""": input_ids,
             """attention_mask""": attention_mask,
         }
         return config, inputs_dict







@require_torch
class        __snake_case			(				__lowerCamelCase  ,     __lowerCamelCase  ,     __lowerCamelCase  ,     unittest.TestCase     ):





       '''simple docstring'''







       lowerCAmelCase__ 						=	(SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
       lowerCAmelCase__ 						=	(SpeechaTextaForCausalLM,) if is_torch_available() else ()
       lowerCAmelCase__ 						=	{"""text-generation""": SpeechaTextaForCausalLM} if is_torch_available() else {}
       lowerCAmelCase__ 						=	True
       lowerCAmelCase__ 						=	False




       def 				UpperCAmelCase__					(    self					:       List[Any]				,							):
         __snake_case:		Tuple					    =		SpeechaTextaStandaloneDecoderModelTester(self				,							is_training=A						)
         __snake_case:		Union[str, Any]					    =		ConfigTester(self				,							config_class=A						)




       def 				UpperCAmelCase__					(    self					:       List[Any]						):
         pass




       def 				UpperCAmelCase__					(    self					:       int						):
         pass




       def 				UpperCAmelCase__					(    self					:       Any						):
         pass




       def 				UpperCAmelCase__					(    self					:       Union[str, Any]						):
         self.config_tester.run_common_tests()




       def 				UpperCAmelCase__					(    self					:       Any						):
         __snake_case:		int					    =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_decoder_model_past(*A						)






       def 				UpperCAmelCase__					(    self					:       List[str]						):
         return













import argparse

import torch

from transformers import NystromformerConfig, NystromformerForMaskedLM







def      A__				(       SCREAMING_SNAKE_CASE__)  ->       Optional[Any]:
  if "model" in orig_key:
    __snake_case:		str					    =		orig_key.replace("""model."""   ,   """""")
  if "norm1" in orig_key:
    __snake_case:		Tuple					    =		orig_key.replace("""norm1"""   ,   """attention.output.LayerNorm""")
  if "norm2" in orig_key:
    __snake_case:		Any					    =		orig_key.replace("""norm2"""   ,   """output.LayerNorm""")
  if "norm" in orig_key:
    __snake_case:		List[str]					    =		orig_key.replace("""norm"""   ,   """LayerNorm""")
  if "transformer" in orig_key:
    __snake_case:		str					    =		orig_key.split(""".""")[0].split("""_""")[-1]
    __snake_case:		int					    =		orig_key.replace(F'''transformer_{layer_num}'''   ,   F'''encoder.layer.{layer_num}''')
  if "mha.attn" in orig_key:
    __snake_case:		int					    =		orig_key.replace("""mha.attn"""   ,   """attention.self""")
  if "mha" in orig_key:
    __snake_case:		List[Any]					    =		orig_key.replace("""mha"""   ,   """attention""")
  if "W_q" in orig_key:
    __snake_case:		Dict					    =		orig_key.replace("""W_q"""   ,   """self.query""")
  if "W_k" in orig_key:
    __snake_case:		str					    =		orig_key.replace("""W_k"""   ,   """self.key""")
  if "W_v" in orig_key:
    __snake_case:		List[Any]					    =		orig_key.replace("""W_v"""   ,   """self.value""")
  if "ff1" in orig_key:
    __snake_case:		int					    =		orig_key.replace("""ff1"""   ,   """intermediate.dense""")
  if "ff2" in orig_key:
    __snake_case:		Dict					    =		orig_key.replace("""ff2"""   ,   """output.dense""")
  if "ff" in orig_key:
    __snake_case:		List[Any]					    =		orig_key.replace("""ff"""   ,   """output.dense""")
  if "mlm_class" in orig_key:
    __snake_case:		Union[str, Any]					    =		orig_key.replace("""mlm.mlm_class"""   ,   """cls.predictions.decoder""")
  if "mlm" in orig_key:
    __snake_case:		List[Any]					    =		orig_key.replace("""mlm"""   ,   """cls.predictions.transform""")
  if "cls" not in orig_key:
    __snake_case:		Dict					    =		"""nystromformer.""" + orig_key

  return orig_key







def      A__				(       SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)  ->       Optional[int]:
  for key in orig_state_dict.copy().keys():
    __snake_case:		str					    =		orig_state_dict.pop(SCREAMING_SNAKE_CASE__)

    if ("pooler" in key) or ("sen_class" in key) or ("conv.bias" in key):
      continue
    else:
      __snake_case:		Tuple					    =		val

  __snake_case:		str					    =		orig_state_dict["""cls.predictions.decoder.bias"""]
  __snake_case:		int					    =		(
      torch.arange(config.max_position_embeddings).expand((1, -1)) + 2
  )

  return orig_state_dict







def      A__				(       SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)  ->       Optional[Any]:
  __snake_case:		int					    =		torch.load(SCREAMING_SNAKE_CASE__   ,   map_location="""cpu""")["""model_state_dict"""]
  __snake_case:		List[Any]					    =		NystromformerConfig.from_json_file(SCREAMING_SNAKE_CASE__)
  __snake_case:		Optional[int]					    =		NystromformerForMaskedLM(SCREAMING_SNAKE_CASE__)

  __snake_case:		Dict					    =		convert_checkpoint_helper(SCREAMING_SNAKE_CASE__   ,   SCREAMING_SNAKE_CASE__)

  model.load_state_dict(SCREAMING_SNAKE_CASE__)
  model.eval()
  model.save_pretrained(SCREAMING_SNAKE_CASE__)

  print(F'''Checkpoint successfuly converted. Model saved at {pytorch_dump_path}''')


if __name__ == "__main__":
      __UpperCAmelCase			:  str										=				argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--pytorch_model_path", default=None, type=str, required=True, help="Path to Nystromformer pytorch checkpoint."
      )
      parser.add_argument(
          "--config_file",
          default=None,
          type=str,
          required=True,
          help="The json file for Nystromformer model config.",
      )
      parser.add_argument(
          "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
      )
      __UpperCAmelCase			:  Optional[Any]										=				parser.parse_args()
      convert_nystromformer_checkpoint(args.pytorch_model_path, args.config_file, args.pytorch_dump_path)






