






import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


_a      	=  {
    '''tiny.en''': '''https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt''',
    '''tiny''': '''https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt''',
    '''base.en''': '''https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt''',
    '''base''': '''https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt''',
    '''small.en''': '''https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt''',
    '''small''': '''https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt''',
    '''medium.en''': '''https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt''',
    '''medium''': '''https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt''',
    '''large''': '''https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt''',
    '''large-v2''': '''https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt''',
}


def __A    (      __lowerCAmelCase	)->  Optional[Any]:





						"""simple docstring"""



						_UpperCAmelCase									=			['layers', 'blocks']
						for k in ignore_keys:
												state_dict.pop(__lowerCAmelCase ,		__lowerCAmelCase	)


_a      	=  {
    '''blocks''': '''layers''',
    '''mlp.0''': '''fc1''',
    '''mlp.2''': '''fc2''',
    '''mlp_ln''': '''final_layer_norm''',
    '''.attn.query''': '''.self_attn.q_proj''',
    '''.attn.key''': '''.self_attn.k_proj''',
    '''.attn.value''': '''.self_attn.v_proj''',
    '''.attn_ln''': '''.self_attn_layer_norm''',
    '''.attn.out''': '''.self_attn.out_proj''',
    '''.cross_attn.query''': '''.encoder_attn.q_proj''',
    '''.cross_attn.key''': '''.encoder_attn.k_proj''',
    '''.cross_attn.value''': '''.encoder_attn.v_proj''',
    '''.cross_attn_ln''': '''.encoder_attn_layer_norm''',
    '''.cross_attn.out''': '''.encoder_attn.out_proj''',
    '''decoder.ln.''': '''decoder.layer_norm.''',
    '''encoder.ln.''': '''encoder.layer_norm.''',
    '''token_embedding''': '''embed_tokens''',
    '''encoder.positional_embedding''': '''encoder.embed_positions.weight''',
    '''decoder.positional_embedding''': '''decoder.embed_positions.weight''',
    '''ln_post''': '''layer_norm''',
}


def __A    (      __lowerCAmelCase	)->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			list(s_dict.keys()	)
						for key in keys:
												_UpperCAmelCase									=			key
												for k, v in WHISPER_MAPPING.items():
																		if k in key:
																								_UpperCAmelCase									=			new_key.replace(__lowerCAmelCase ,		__lowerCAmelCase	)

												print(F"""{key} -> {new_key}"""	)

												_UpperCAmelCase									=			s_dict.pop(__lowerCAmelCase	)
						return s_dict


def __A    (      __lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						_UpperCAmelCase					, _UpperCAmelCase									=			emb.weight.shape
						_UpperCAmelCase									=			nn.Linear(__lowerCAmelCase ,		__lowerCAmelCase ,		bias=__lowerCAmelCase	)
						_UpperCAmelCase									=			emb.weight.data
						return lin_layer


def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  bytes:





						"""simple docstring"""



						os.makedirs(__lowerCAmelCase ,		exist_ok=__lowerCAmelCase	)
						_UpperCAmelCase									=			os.path.basename(__lowerCAmelCase	)

						_UpperCAmelCase									=			url.split('/'	)[-2]
						_UpperCAmelCase									=			os.path.join(__lowerCAmelCase ,		__lowerCAmelCase	)

						if os.path.exists(__lowerCAmelCase	) and not os.path.isfile(__lowerCAmelCase	):
												raise RuntimeError(F"""{download_target} exists and is not a regular file"""	)

						if os.path.isfile(__lowerCAmelCase	):
												_UpperCAmelCase									=			open(__lowerCAmelCase ,		'rb'	).read()
												if hashlib.shaaaa(__lowerCAmelCase	).hexdigest() == expected_shaaaa:
																		return model_bytes
												else:
																		warnings.warn(F"""{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"""	)

						with urllib.request.urlopen(__lowerCAmelCase	) as source, open(__lowerCAmelCase ,		'wb'	) as output:
												with tqdm(
												    total=int(source.info().get('Content-Length'	)	) ,		ncols=80 ,		unit='iB' ,		unit_scale=__lowerCAmelCase ,		unit_divisor=1_024	) as loop:
																		while True:
																								_UpperCAmelCase									=			source.read(8_192	)
																								if not buffer:
																														break

																								output.write(__lowerCAmelCase	)
																								loop.update(len(__lowerCAmelCase	)	)

						_UpperCAmelCase									=			open(__lowerCAmelCase ,		'rb'	).read()
						if hashlib.shaaaa(__lowerCAmelCase	).hexdigest() != expected_shaaaa:
												raise RuntimeError(
												    'Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model.'	)

						return model_bytes


def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Optional[int]:





						"""simple docstring"""



						if ".pt" not in checkpoint_path:
												_UpperCAmelCase									=			_download(_MODELS[checkpoint_path]	)
						else:
												_UpperCAmelCase									=			torch.load(__lowerCAmelCase ,		map_location='cpu'	)
						_UpperCAmelCase									=			original_checkpoint['dims']
						_UpperCAmelCase									=			original_checkpoint['model_state_dict']
						_UpperCAmelCase									=			state_dict['decoder.token_embedding.weight']
						remove_ignore_keys_(__lowerCAmelCase	)
						rename_keys(__lowerCAmelCase	)
						_UpperCAmelCase									=			True
						_UpperCAmelCase									=			state_dict['decoder.layers.0.fc1.weight'].shape[0]

						_UpperCAmelCase									=			WhisperConfig(
						    vocab_size=dimensions['n_vocab'] ,		encoder_ffn_dim=__lowerCAmelCase ,		decoder_ffn_dim=__lowerCAmelCase ,		num_mel_bins=dimensions['n_mels'] ,		d_model=dimensions['n_audio_state'] ,		max_target_positions=dimensions['n_text_ctx'] ,		encoder_layers=dimensions['n_audio_layer'] ,		encoder_attention_heads=dimensions['n_audio_head'] ,		decoder_layers=dimensions['n_text_layer'] ,		decoder_attention_heads=dimensions['n_text_state'] ,		max_source_positions=dimensions['n_audio_ctx'] ,		)

						_UpperCAmelCase									=			WhisperForConditionalGeneration(__lowerCAmelCase	)
						_UpperCAmelCase					, _UpperCAmelCase									=			model.model.load_state_dict(__lowerCAmelCase ,		strict=__lowerCAmelCase	)
						if len(__lowerCAmelCase	) > 0 and not set(__lowerCAmelCase	) <= {
						    "encoder.embed_positions.weights",
						    "decoder.embed_positions.weights",
						}:
												raise ValueError(
												    'Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,'
												    F""" but all the following weights are missing {missing}"""	)

						if tie_embeds:
												_UpperCAmelCase									=			make_linear_from_emb(model.model.decoder.embed_tokens	)
						else:
												_UpperCAmelCase									=			proj_out_weights

						model.save_pretrained(__lowerCAmelCase	)


if __name__ == "__main__":
		_a      	=  argparse.ArgumentParser()
		# # Required parameters
		parser.add_argument('''--checkpoint_path''', type=str, help='''Patht to the downloaded checkpoints''')
		parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
		_a      	=  parser.parse_args()

		convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)







"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__					=   "▁"

SCREAMING_SNAKE_CASE__					=   {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__					=   {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

SCREAMING_SNAKE_CASE__					=   {
    "google/pegasus-xsum": 512,
}


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)

class 		lowercase					(					_UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE					=      VOCAB_FILES_NAMES

      _SCREAMING_SNAKE_CASE					=      VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE					=      ['input_ids', 'attention_mask']





      def __init__(       self      ,						lowercase      ,						lowercase="<pad>"      ,						lowercase="</s>"      ,						lowercase="<unk>"      ,						lowercase="<mask_2>"      ,						lowercase="<mask_1>"      ,						lowercase=None      ,						lowercase=103      ,						lowercase = None      ,						**lowercase      ,						)		->     None:
            lowerCAmelCase  						=       offset
            if additional_special_tokens is not None:
                  if not isinstance(lowercase      ,						lowercase ):
                        raise TypeError(
                            f'additional_special_tokens should be of type {type(lowercase )}, but is'
                            f' {type(lowercase )}' )

                  lowerCAmelCase  						=       (
                      ([mask_token_sent] + additional_special_tokens)
                      if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                      else additional_special_tokens
                  )
                  # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                  additional_special_tokens_extended += [
                      f'<unk_{i}>' for i in range(len(lowercase )      ,						self.offset - 1 )
                  ]

                  if len(set(lowercase ) ) != len(lowercase ):
                        raise ValueError(
                            """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
                            f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.' )
                  lowerCAmelCase  						=       additional_special_tokens_extended
            else:
                  lowerCAmelCase  						=       [mask_token_sent] if mask_token_sent is not None else []
                  additional_special_tokens += [f'<unk_{i}>' for i in range(2      ,						self.offset )]

            lowerCAmelCase  						=       {} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                eos_token=lowercase      ,						unk_token=lowercase      ,						mask_token=lowercase      ,						pad_token=lowercase      ,						mask_token_sent=lowercase      ,						offset=lowercase      ,						additional_special_tokens=lowercase      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**lowercase      ,						)
            lowerCAmelCase  						=       mask_token_sent
            lowerCAmelCase  						=       vocab_file
            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(lowercase )

            # add special tokens to encoder dict
            lowerCAmelCase  						=       {
                0: self.pad_token,
                1: self.eos_token,
            }

            if self.mask_token_sent is not None:
                  self.encoder.update(
                      {
                          2: self.mask_token_sent,
                          3: self.mask_token,
                      } )

            if self.offset > 0:
                  # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
                  # mask_token_sent is already added to list -> so start at 1
                  self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,						self.offset - 1 )} )

            lowerCAmelCase  						=       {v: k for k, v in self.encoder.items()}





      @property
      def     _snake_case			(       self )		->     int:
            return len(self.sp_model ) + self.offset





      def     _snake_case			(       self )		->     Dict[str, int]:
            lowerCAmelCase  						=       {self.convert_ids_to_tokens(lowercase ): i for i in range(self.vocab_size )}
            vocab.update(self.added_tokens_encoder )
            return vocab





      def __getstate__(       self )		->     Optional[int]:
            lowerCAmelCase  						=       self.__dict__.copy()
            lowerCAmelCase  						=       None
            return state





      def __setstate__(       self      ,						lowercase )		->     List[Any]:
            lowerCAmelCase  						=       d

            # for backward compatibility
            if not hasattr(self      ,						"""sp_model_kwargs""" ):
                  lowerCAmelCase  						=       {}

            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(self.vocab_file )





      def     _snake_case			(       self      ,						lowercase )		->     List[str]:
            return self.sp_model.encode(lowercase      ,						out_type=lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     int:
            if token in self.decoder:
                  return self.decoder[token]
            elif token in self.added_tokens_decoder:
                  return self.added_tokens_decoder[token]
            lowerCAmelCase  						=       self.sp_model.piece_to_id(lowercase )
            return sp_id + self.offset





      def     _snake_case			(       self      ,						lowercase )		->     str:
            if index in self.encoder:
                  return self.encoder[index]
            elif index in self.added_tokens_encoder:
                  return self.added_tokens_encoder[index]
            else:
                  lowerCAmelCase  						=       self.sp_model.IdToPiece(index - self.offset )
            return token





      def     _snake_case			(       self      ,						lowercase )		->     Optional[int]:
            lowerCAmelCase  						=       []
            lowerCAmelCase  						=       """"""
            for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                        out_string += self.sp_model.decode(lowercase ) + token
                        lowerCAmelCase  						=       []
                  else:
                        current_sub_tokens.append(lowercase )
            out_string += self.sp_model.decode(lowercase )
            return out_string.strip()





      def     _snake_case			(       self      ,						lowercase=False )		->     Tuple:
            return 1





      def     _snake_case			(       self      ,						lowercase )		->     Tuple:
            lowerCAmelCase  						=       set(self.all_special_ids )  # call it once instead of inside list comp
            all_special_ids.remove(self.unk_token_id )  # <unk> is only sometimes special

            return [1 if x in all_special_ids else 0 for x in seq]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None      ,						lowercase = False )		->     List[int]:
            if already_has_special_tokens:
                  return self._special_token_mask(lowercase )
            elif token_ids_a is None:
                  return self._special_token_mask(lowercase ) + [1]
            else:
                  return self._special_token_mask(token_ids_a + token_ids_a ) + [1]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase=None )		->     List[int]:
            if token_ids_a is None:
                  return token_ids_a + [self.eos_token_id]
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return token_ids_a + token_ids_a + [self.eos_token_id]





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None )		->     Tuple[str]:
            if not os.path.isdir(lowercase ):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
                  return
            lowerCAmelCase  						=       os.path.join(
                lowercase      ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

            if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase ) and os.path.isfile(self.vocab_file ):
                  copyfile(self.vocab_file      ,						lowercase )
            elif not os.path.isfile(self.vocab_file ):
                  with open(lowercase      ,						"""wb""" ) as fi:
                        lowerCAmelCase  						=       self.sp_model.serialized_model_proto()
                        fi.write(lowercase )

            return (out_vocab_file,)






