


"""simple docstring"""




import argparse
import json
import os

import fairseq
import torch
from torch import nn

from transformers import (
    SpeechaTextaConfig,
    SpeechaTextaForCausalLM,
    SpeechaTextaTokenizer,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
lowercase_			     =  logging.get_logger(__name__)

lowercase_			     =  {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.k_proj": "encoder.layers.*.attention.k_proj",
    "self_attn.v_proj": "encoder.layers.*.attention.v_proj",
    "self_attn.q_proj": "encoder.layers.*.attention.q_proj",
    "self_attn.out_proj": "encoder.layers.*.attention.out_proj",
    "self_attn_layer_norm": "encoder.layers.*.layer_norm",
    "fc1": "encoder.layers.*.feed_forward.intermediate_dense",
    "fc2": "encoder.layers.*.feed_forward.output_dense",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "quantizer.weight_proj": "quantizer.weight_proj",
    "quantizer.vars": "quantizer.codevectors",
    "project_q": "project_q",
    "final_proj": "project_hid",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}
lowercase_			     =  [
    "lm_head",
    "quantizer.weight_proj",
    "quantizer.codevectors",
    "project_q",
    "project_hid",
]

def        lowercase							(		lowerCAmelCase__ :     Any    ,					lowerCAmelCase__ :     Optional[int]    ,					lowerCAmelCase__ :     List[str]    ,					lowerCAmelCase__ :     Tuple    ,					lowerCAmelCase__ :     int			) ->      Union[str, Any]:
		for attribute in key.split('''.'''			):
				__a           =			getattr(lowerCAmelCase__    ,					lowerCAmelCase__			)

		if weight_type is not None:
				__a           =			getattr(lowerCAmelCase__    ,					lowerCAmelCase__			).shape
		else:
				__a           =			hf_pointer.shape

		assert hf_shape == value.shape, (
		    f'''Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be'''
		    f''' {value.shape} for {full_name}'''
		)

		if weight_type == "weight":
				__a           =			value
		elif weight_type == "weight_g":
				__a           =			value
		elif weight_type == "weight_v":
				__a           =			value
		elif weight_type == "bias":
				__a           =			value
		else:
				__a           =			value

		logger.info(f'''{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.'''			)

def        lowercase							(		lowerCAmelCase__ :     List[str]    ,					lowerCAmelCase__ :     int			) ->      int:
		__a           =			[]
		__a           =			fairseq_model.state_dict()

		__a           =			hf_model.feature_extractor

		# if encoder has different dim to decoder -> use proj_weight
		__a           =			None

		for name, value in fairseq_dict.items():
				__a           =			False
				if "conv_layers" in name:
						load_conv_layer(
						    lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__    ,					hf_model.config.feat_extract_norm == '''group'''    ,					)
						__a           =			True
				elif name.split('''.'''			)[0] == "proj":
						__a           =			fairseq_model.proj
						__a           =			True
				else:
						for key, mapped_key in MAPPING.items():
								if key in name or key.split('''w2v_model.'''			)[-1] == name.split('''.'''			)[0]:
										__a           =			True
										if "*" in mapped_key:
												__a           =			name.split(lowerCAmelCase__			)[0].split('''.'''			)[-2]
												__a           =			mapped_key.replace('''*'''    ,					lowerCAmelCase__			)
										if "weight_g" in name:
												__a           =			'''weight_g'''
										elif "weight_v" in name:
												__a           =			'''weight_v'''
										elif "bias" in name:
												__a           =			'''bias'''
										elif "weight" in name:
												__a           =			'''weight'''
										else:
												__a           =			None
										set_recursively(lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__			)
								continue
				if not is_used:
						unused_weights.append(lowerCAmelCase__			)

		logger.warning(f'''Unused weights: {unused_weights}'''			)

		return proj_weight

def        lowercase							(		lowerCAmelCase__ :     str    ,					lowerCAmelCase__ :     Optional[Any]    ,					lowerCAmelCase__ :     str    ,					lowerCAmelCase__ :     str    ,					lowerCAmelCase__ :     Any			) ->      str:
		__a           =			full_name.split('''conv_layers.'''			)[-1]
		__a           =			name.split('''.'''			)
		__a           =			int(items[0]			)
		__a           =			int(items[1]			)

		if type_id == 0:
				if "bias" in name:
						assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
						    f'''{full_name} has size {value.shape}, but'''
						    f''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
						)
						__a           =			value
						logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''			)
				elif "weight" in name:
						assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
						    f'''{full_name} has size {value.shape}, but'''
						    f''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
						)
						__a           =			value
						logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''			)
		elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
				if "bias" in name:
						assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
						    f'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
						    " found."
						)
						__a           =			value
						logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''			)
				elif "weight" in name:
						assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
						    f'''{full_name} has size {value.shape}, but'''
						    f''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
						)
						__a           =			value
						logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''			)
		else:
				unused_weights.append(lowerCAmelCase__			)

def        lowercase							(		lowerCAmelCase__ :     int			) ->      List[str]:
		__a      ,     __a           =			emb.weight.shape
		__a           =			nn.Linear(lowerCAmelCase__    ,					lowerCAmelCase__    ,					bias=lowerCAmelCase__			)
		__a           =			emb.weight.data
		return lin_layer

def        lowercase							(		lowerCAmelCase__ :     Optional[Any]			) ->      Tuple:
		with open(lowerCAmelCase__    ,					'''r'''    ,					encoding='''utf-8'''			) as f:
				__a           =			f.readlines()
				__a           =			[line.split(''' '''			)[0] for line in lines]

		__a           =			len(lowerCAmelCase__			)

		__a           =			{
		    '''<s>''': 0,
		    '''<pad>''': 1,
		    '''</s>''': 2,
		    '''<unk>''': 3,
		}

		vocab_dict.update(dict(zip(lowerCAmelCase__    ,					range(4    ,					num_words + 4			)			)			)			)
		return vocab_dict

@torch.no_grad()
def        lowercase							(		lowerCAmelCase__ :     List[Any]    ,					lowerCAmelCase__ :     int    ,					lowerCAmelCase__ :     Optional[int]    ,					lowerCAmelCase__ :     str    ,					lowerCAmelCase__ :     Optional[int]    ,					lowerCAmelCase__ :     int    ,					lowerCAmelCase__ :     Union[str, Any]    ,					) ->      Union[str, Any]:
		__a           =			WavaVecaConfig.from_pretrained(lowerCAmelCase__			)
		__a           =			SpeechaTextaConfig.from_pretrained(
		    lowerCAmelCase__    ,					vocab_size=lowerCAmelCase__    ,					decoder_layers=lowerCAmelCase__    ,					do_stable_layer_norm=lowerCAmelCase__			)

		__a           =			WavaVecaFeatureExtractor(
		    feature_size=1    ,					sampling_rate=16000    ,					padding_value=0    ,					do_normalize=lowerCAmelCase__    ,					return_attention_mask=lowerCAmelCase__    ,					)

		__a      ,     __a      ,     __a           =			fairseq.checkpoint_utils.load_model_ensemble_and_task(
		    [checkpoint_path]    ,					arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''			)[:-1]			)}			)
		__a           =			model[0].eval()

		# set weights for wav2vec2 encoder
		__a           =			WavaVecaModel(lowerCAmelCase__			)
		__a           =			recursively_load_weights_wavaveca(model.encoder    ,					lowerCAmelCase__			)

		__a           =			SpeechaTextaForCausalLM(lowerCAmelCase__			)
		__a      ,     __a           =			hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict()    ,					strict=lowerCAmelCase__			)

		# set output linear layer
		unexpected_keys.remove('''embed_out'''			)
		__a           =			nn.Parameter(model.decoder.embed_out.detach()			)

		# layer norm is init to identity matrix so leaving it is fine
		logger.warning(f'''The following keys are missing when loading the decoder weights: {missing_keys}'''			)
		logger.warning(f'''The following keys are unexpected when loading the decoder weights: {unexpected_keys}'''			)

		__a           =			SpeechEncoderDecoderModel(encoder=lowerCAmelCase__    ,					decoder=lowerCAmelCase__			)
		__a           =			False

		# add projection layer
		__a           =			nn.Parameter(projection_layer.weight			)
		__a           =			nn.Parameter(projection_layer.bias			)

		__a           =			create_vocab_dict(lowerCAmelCase__			)

		with open(os.path.join(lowerCAmelCase__    ,					'''vocab.json'''			)    ,					'''w'''			) as fp:
				json.dump(lowerCAmelCase__    ,					lowerCAmelCase__			)

		__a           =			SpeechaTextaTokenizer(os.path.join(lowerCAmelCase__    ,					'''vocab.json'''			)			)
		tokenizer.save_pretrained(lowerCAmelCase__			)

		__a           =			hf_wavavec.config.to_dict()
		__a           =			tokenizer.pad_token_id
		__a           =			tokenizer.bos_token_id
		__a           =			tokenizer.eos_token_id
		__a           =			'''speech_to_text_2'''
		__a           =			'''wav2vec2'''

		__a           =			SpeechEncoderDecoderConfig.from_dict(lowerCAmelCase__			)

		hf_wavavec.save_pretrained(lowerCAmelCase__			)
		feature_extractor.save_pretrained(lowerCAmelCase__			)


if __name__ == "__main__":
				lowercase_			     =  argparse.ArgumentParser()
				parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
				parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
				parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
				parser.add_argument(
				    "--encoder_config_path",
				    default="facebook/wav2vec2-large-lv60",
				    type=str,
				    help="Path to hf encoder wav2vec2 checkpoint config",
				)
				parser.add_argument(
				    "--decoder_config_path",
				    default="facebook/s2t-small-mustc-en-fr-st",
				    type=str,
				    help="Path to hf decoder s2t checkpoint config",
				)
				parser.add_argument("--vocab_size", default=1_0_2_2_4, type=int, help="Vocab size of decoder")
				parser.add_argument("--num_decoder_layers", default=7, type=int, help="Number of decoder layers")

				lowercase_			     =  parser.parse_args()
				convert_wavaveca_checkpoint(
				    args.checkpoint_path,
				    args.pytorch_dump_folder_path,
				    args.dict_path,
				    encoder_config_path=args.encoder_config_path,
				    decoder_config_path=args.decoder_config_path,
				    vocab_size=args.vocab_size,
				    num_decoder_layers=args.num_decoder_layers,
				)










"""simple docstring"""




import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    CTRLForSequenceClassification,
				    CTRLLMHeadModel,
				    CTRLModel,
				)
class 					__lowerCAmelCase					:


		'''simple docstring'''



		def __init__(  self							, _a							, _a=14							, _a=7							, _a=True							, _a=True							, _a=True							, _a=True							, _a=True							, _a=99							, _a=32							, _a=5							, _a=4							, _a=37							, _a="gelu"							, _a=0.1							, _a=0.1							, _a=512							, _a=16							, _a=2							, _a=0.02							, _a=3							, _a=4							, _a=None							, ):
				__a           =			parent
				__a           =			batch_size
				__a           =			seq_length
				__a           =			is_training
				__a           =			use_token_type_ids
				__a           =			use_input_mask
				__a           =			use_labels
				__a           =			use_mc_token_ids
				__a           =			vocab_size
				__a           =			hidden_size
				__a           =			num_hidden_layers
				__a           =			num_attention_heads
				__a           =			intermediate_size
				__a           =			hidden_act
				__a           =			hidden_dropout_prob
				__a           =			attention_probs_dropout_prob
				__a           =			max_position_embeddings
				__a           =			type_vocab_size
				__a           =			type_sequence_label_size
				__a           =			initializer_range
				__a           =			num_labels
				__a           =			num_choices
				__a           =			scope
				__a           =			self.vocab_size - 1

		def 			__UpperCAmelCase       (  self      ):
				__a           =			ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size      )

				__a           =			None
				if self.use_input_mask:
						__a           =			random_attention_mask([self.batch_size, self.seq_length]      )

				__a           =			None
				if self.use_token_type_ids:
						__a           =			ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size      )

				__a           =			None
				if self.use_mc_token_ids:
						__a           =			ids_tensor([self.batch_size, self.num_choices]							, self.seq_length      )

				__a           =			None
				__a           =			None
				__a           =			None
				if self.use_labels:
						__a           =			ids_tensor([self.batch_size]							, self.type_sequence_label_size      )
						__a           =			ids_tensor([self.batch_size, self.seq_length]							, self.num_labels      )
						__a           =			ids_tensor([self.batch_size]							, self.num_choices      )

				__a           =			self.get_config()

				__a           =			ids_tensor([self.num_hidden_layers, self.num_attention_heads]							, 2      )

				return (
				    config,
				    input_ids,
				    input_mask,
				    head_mask,
				    token_type_ids,
				    mc_token_ids,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				)

		def 			__UpperCAmelCase       (  self      ):
				return CTRLConfig(
				    vocab_size=self.vocab_size							, n_embd=self.hidden_size							, n_layer=self.num_hidden_layers							, n_head=self.num_attention_heads							, n_positions=self.max_position_embeddings							, pad_token_id=self.pad_token_id							, )

		def 			__UpperCAmelCase       (  self							, _a							, _a							, _a							, _a							, _a							, *_a      ):
				__a           =			CTRLModel(config=_a      )
				model.to(_a      )
				model.eval()

				model(_a							, token_type_ids=_a							, head_mask=_a      )
				model(_a							, token_type_ids=_a      )
				__a           =			model(_a      )
				self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.hidden_size)      )
				self.parent.assertEqual(len(result.past_key_values      )							, config.n_layer      )

		def 			__UpperCAmelCase       (  self							, _a							, _a							, _a							, _a							, _a							, *_a      ):
				__a           =			CTRLLMHeadModel(_a      )
				model.to(_a      )
				model.eval()

				__a           =			model(_a							, token_type_ids=_a							, labels=_a      )
				self.parent.assertEqual(result.loss.shape							, ()      )
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			self.prepare_config_and_inputs()

				(
				    (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     (
				    __a
				)      ,     
				)           =			config_and_inputs

				__a           =			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''head_mask''': head_mask}

				return config, inputs_dict





		def 			__UpperCAmelCase       (  self							, _a							, _a							, _a							, _a							, *_a      ):
				__a           =			self.num_labels
				__a           =			CTRLForSequenceClassification(_a      )
				model.to(_a      )
				model.eval()
				__a           =			ids_tensor([self.batch_size]							, self.type_sequence_label_size      )
				__a           =			model(_a							, token_type_ids=_a							, labels=_a      )
				self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.num_labels)      )







@require_torch
class 					__lowerCAmelCase					(   __SCREAMING_SNAKE_CASE     ,			__SCREAMING_SNAKE_CASE     ,			__SCREAMING_SNAKE_CASE     ,			unittest.TestCase       ):


		'''simple docstring'''



		__UpperCAmelCase		:			str										=				(CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
		__UpperCAmelCase		:			Union[str, Any]										=				(CTRLLMHeadModel,) if is_torch_available() else ()
		__UpperCAmelCase		:			Union[str, Any]										=				(
		    {
		        'feature-extraction': CTRLModel,
		        'text-classification': CTRLForSequenceClassification,
		        'text-generation': CTRLLMHeadModel,
		        'zero-shot': CTRLForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__UpperCAmelCase		:			Optional[Any]										=				True
		__UpperCAmelCase		:			List[Any]										=				False
		__UpperCAmelCase		:			str										=				False

		def 			__UpperCAmelCase       (  self							, _a							, _a							, _a							, _a							, _a      ):
				if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
						# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
						# `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
						# config could not be created.
						return True

				return False

		def 			__UpperCAmelCase       (  self      ):
				__a           =			CTRLModelTester(self      )
				__a           =			ConfigTester(self							, config_class=_a							, n_embd=37      )

		def 			__UpperCAmelCase       (  self      ):
				super().tearDown()
				# clean-up as much as possible GPU memory occupied by PyTorch
				gc.collect()
				torch.cuda.empty_cache()

		def 			__UpperCAmelCase       (  self      ):
				self.config_tester.run_common_tests()

		def 			__UpperCAmelCase       (  self      ):
				__a           =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_ctrl_model(*_a      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_lm_head_model(*_a      )

		@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''      )
		def 			__UpperCAmelCase       (  self      ):
				pass

		@slow
		def 			__UpperCAmelCase       (  self      ):
				for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__a           =			CTRLModel.from_pretrained(_a      )
						self.assertIsNotNone(_a      )





		@unittest.skip('''The model doesn\'t support left padding'''      )  # and it's not used enough to be worth fixing :)
		def 			__UpperCAmelCase       (  self      ):
				pass







@require_torch
class 					__lowerCAmelCase					(   unittest.TestCase       ):


		'''simple docstring'''



		def 			__UpperCAmelCase       (  self      ):
				super().tearDown()
				# clean-up as much as possible GPU memory occupied by PyTorch
				gc.collect()
				torch.cuda.empty_cache()





		@slow
		def 			__UpperCAmelCase       (  self      ):
				__a           =			CTRLLMHeadModel.from_pretrained('''ctrl'''      )
				model.to(_a      )
				__a           =			torch.tensor(
				    [[11_859, 0, 1_611, 8]]							, dtype=torch.long							, device=_a      )  # Legal the president is
				__a           =			[
				    11_859,
				    0,
				    1_611,
				    8,
				    5,
				    150,
				    26_449,
				    2,
				    19,
				    348,
				    469,
				    3,
				    2_595,
				    48,
				    20_740,
				    246_533,
				    246_533,
				    19,
				    30,
				    5,
				]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

				__a           =			model.generate(_a							, do_sample=_a      )
				self.assertListEqual(output_ids[0].tolist()							, _a      )






