





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig


def        A__						(							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_=0		) ->  Union[str, Any]:
		# Format the message.
		if name is None:
				lowerCamelCase :			List[str]    		=None
		else:
				lowerCamelCase :			str    		='''.''' * max(0				,							spaces - 2		) + '''# {:''' + str(5_0 - spaces		) + '''s}'''
				lowerCamelCase :			str    		=fmt.format(SCREAMING_SNAKE_CASE_		)

		# Print and recurse (if needed).
		if isinstance(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		):
				if msg is not None:
						print(SCREAMING_SNAKE_CASE_		)
				for k in val.keys():
						recursive_print(SCREAMING_SNAKE_CASE_				,							val[k]				,							spaces + 2		)
		elif isinstance(SCREAMING_SNAKE_CASE_				,							torch.Tensor		):
				print(SCREAMING_SNAKE_CASE_				,							''':'''				,							val.size()		)
		else:
				print(SCREAMING_SNAKE_CASE_				,							''':'''				,							SCREAMING_SNAKE_CASE_		)


def        A__						(							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		) ->  List[Any]:
		# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
		# for compatibility with later versions of NVIDIA Megatron-LM.
		# The inverse operation is performed inside Megatron-LM to read checkpoints:
		# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
		# If param is the weight tensor of the self-attention block, the returned tensor
		# will have to be transposed one more time to be read by HuggingFace BERT.
		lowerCamelCase :			Tuple    		=param.size()
		if checkpoint_version == 1.0:
				# version 1.0 stores [num_heads * hidden_size * num_splits, :]
				lowerCamelCase :			List[str]    		=(num_heads, hidden_size, num_splits) + input_shape[1:]
				lowerCamelCase :			Any    		=param.view(*SCREAMING_SNAKE_CASE_		)
				lowerCamelCase :			str    		=param.transpose(0				,							2		)
				lowerCamelCase :			Union[str, Any]    		=param.transpose(1				,							2		).contiguous()
		elif checkpoint_version >= 2.0:
				# other versions store [num_heads * num_splits * hidden_size, :]
				lowerCamelCase :			Optional[int]    		=(num_heads, num_splits, hidden_size) + input_shape[1:]
				lowerCamelCase :			Optional[Any]    		=param.view(*SCREAMING_SNAKE_CASE_		)
				lowerCamelCase :			List[str]    		=param.transpose(0				,							1		).contiguous()
		lowerCamelCase :			Union[str, Any]    		=param.view(*SCREAMING_SNAKE_CASE_		)
		return param


def        A__						(							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		) ->  Union[str, Any]:
		# The converted output model.
		lowerCamelCase :			Optional[Any]    		={}

		# old versions did not store training args
		lowerCamelCase :			List[str]    		=input_state_dict.get('''args'''				,							SCREAMING_SNAKE_CASE_		)
		if ds_args is not None:
				# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
				# from pprint import pprint
				# pprint(vars(ds_args))

				lowerCamelCase :			Optional[Any]    		=ds_args.tokenizer_type
				lowerCamelCase :			Tuple    		=ds_args.padded_vocab_size
				lowerCamelCase :			Optional[Any]    		=ds_args.max_position_embeddings
				lowerCamelCase :			str    		=ds_args.hidden_size
				lowerCamelCase :			int    		=ds_args.num_layers
				lowerCamelCase :			Tuple    		=ds_args.num_attention_heads
				lowerCamelCase :			Union[str, Any]    		=ds_args.ffn_hidden_size if '''ffn_hidden_size''' in ds_args else 4 * ds_args.hidden_size
				# pprint(config)

		# The number of heads.
		lowerCamelCase :			Optional[int]    		=config.num_attention_heads
		# The hidden_size per head.
		lowerCamelCase :			Tuple    		=config.hidden_size // heads
		# Megatron-LM checkpoint version
		if "checkpoint_version" in input_state_dict.keys():
				lowerCamelCase :			Dict    		=input_state_dict['''checkpoint_version''']
		else:
				lowerCamelCase :			Any    		=0.0

		# The model.
		lowerCamelCase :			Any    		=input_state_dict['''model''']
		# The language model.
		lowerCamelCase :			Dict    		=model['''language_model''']
		# The embeddings.
		lowerCamelCase :			Tuple    		=lm['''embedding''']

		# The word embeddings.
		lowerCamelCase :			List[Any]    		=embeddings['''word_embeddings''']['''weight''']
		# Truncate the embedding table to vocab_size rows.
		lowerCamelCase :			List[str]    		=word_embeddings[: config.vocab_size, :]
		# Store the word embeddings.
		lowerCamelCase :			Dict    		=word_embeddings

		# The position embeddings.
		lowerCamelCase :			int    		=embeddings['''position_embeddings''']['''weight''']
		assert pos_embeddings.size(0		) == config.max_position_embeddings and pos_embeddings.size(1		) == config.hidden_size
		# Store the position embeddings.
		lowerCamelCase :			Union[str, Any]    		=pos_embeddings

		# The token-type embeddings.
		lowerCamelCase :			List[Any]    		=embeddings['''tokentype_embeddings''']['''weight''']
		# Store the position embeddings.
		lowerCamelCase :			Any    		=tokentype_embeddings

		# The transformer.
		lowerCamelCase :			int    		=lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

		# The regex to extract layer names.
		lowerCamelCase :			Dict    		=re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''		)

		# The simple map of names for "automated" rules.
		lowerCamelCase :			Optional[int]    		={
		    '''attention.dense''': '''.attention.output.dense.''',
		    '''self_attention.dense''': '''.attention.output.dense.''',
		    '''mlp.dense_h_to_4h''': '''.intermediate.dense.''',
		    '''mlp.dense_4h_to_h''': '''.output.dense.''',
		}

		# Keep track of the attention/query/value tensor.
		lowerCamelCase :			int    		=None

		# Extract the layers.
		for key, val in transformer.items():
				# Match the name.
				lowerCamelCase :			str    		=layer_re.match(SCREAMING_SNAKE_CASE_		)

				# Stop if that's not a layer
				if m is None:
						break

				# The index of the layer.
				lowerCamelCase :			List[Any]    		=int(m.group(1		)		)
				# The name of the operation.
				lowerCamelCase :			List[str]    		=m.group(2		)
				# Is it a weight or a bias?
				lowerCamelCase :			Union[str, Any]    		=m.group(3		)

				# The name of the layer.
				lowerCamelCase :			List[Any]    		=F"bert.encoder.layer.{layer_idx}"

				# For layernorm(s), simply store the layer norm.
				if op_name.endswith('''layernorm'''		):
						lowerCamelCase :			Optional[Any]    		='''attention.ln''' if op_name.startswith('''input'''		) else '''ln'''
						lowerCamelCase :			Tuple    		=val

				# Transpose the QKV matrix.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "weight":
						# Make sure the QKV pointer is nil.
						assert attention_qkv_weight is None, ""

						lowerCamelCase :			str    		=fix_query_key_value_ordering(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							3				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
						# Store the tensor as we need the bias as well to interleave QKV and biases.
						lowerCamelCase :			Tuple    		=out_val

				# Transpose the bias.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "bias":
						# Make sure we read the weight tensor.
						assert attention_qkv_weight is not None, ""

						# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
						lowerCamelCase :			Optional[int]    		=attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
						lowerCamelCase :			Tuple    		=attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
						lowerCamelCase :			Dict    		=attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

						lowerCamelCase :			str    		=fix_query_key_value_ordering(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							3				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
						# Split the bias.
						lowerCamelCase :			int    		=out_val[0 * config.hidden_size : 1 * config.hidden_size]
						lowerCamelCase :			Optional[Any]    		=out_val[1 * config.hidden_size : 2 * config.hidden_size]
						lowerCamelCase :			Optional[Any]    		=out_val[2 * config.hidden_size : 3 * config.hidden_size]

						# Store.
						lowerCamelCase :			Optional[Any]    		=q
						lowerCamelCase :			Any    		=q_bias
						lowerCamelCase :			int    		=k
						lowerCamelCase :			Optional[int]    		=k_bias
						lowerCamelCase :			List[Any]    		=v
						lowerCamelCase :			Optional[int]    		=v_bias

						# Clear the stored tensor.
						lowerCamelCase :			Union[str, Any]    		=None

				# Copy weights and biases as is.
				elif weight_or_bias in ["weight", "bias"]:
						lowerCamelCase :			Optional[int]    		=megatron_to_transformers[op_name]
						lowerCamelCase :			Tuple    		=val

    # The final layernorm.
		lowerCamelCase :			Optional[int]    		=transformer['''final_layernorm.weight''']
		lowerCamelCase :			Any    		=transformer['''final_layernorm.bias''']

		# The pooler.
		lowerCamelCase :			Optional[Any]    		=lm['''pooler''']

		# Store the matrix and the bias.
		lowerCamelCase :			List[Any]    		=pooler['''dense.weight''']
		lowerCamelCase :			Union[str, Any]    		=pooler['''dense.bias''']

		# The LM head from Megatron (for RACE).
		lowerCamelCase :			List[Any]    		=model['''lm_head''']

		# The transform matrix.
		lowerCamelCase :			List[Any]    		=lm_head['''dense.weight''']
		lowerCamelCase :			List[Any]    		=lm_head['''dense.bias''']

		# The transform LN.
		lowerCamelCase :			List[Any]    		=lm_head['''layernorm.weight''']
		lowerCamelCase :			Optional[int]    		=lm_head['''layernorm.bias''']

		# For the decoder, we replicate the weights.
		lowerCamelCase :			Any    		=word_embeddings
		lowerCamelCase :			int    		=lm_head['''bias''']

		# The classifier from Megatron (for MLNI).
		lowerCamelCase :			List[str]    		=model['''binary_head''']

		# Store the classifier.
		lowerCamelCase :			Dict    		=binary_head['''weight''']
		lowerCamelCase :			Any    		=binary_head['''bias''']

		# It should be done!
		return output_state_dict


def        A__						(							) ->  List[Any]:
		# Create the argument parser.
		lowerCamelCase :			Any    		=argparse.ArgumentParser()
		parser.add_argument('''--print-checkpoint-structure'''				,							action='''store_true'''		)
		parser.add_argument('''path_to_checkpoint'''				,							type=SCREAMING_SNAKE_CASE_				,							help='''Path to the ZIP file containing the checkpoint'''		)
		parser.add_argument(
		    '''--config_file'''				,							default=''''''				,							type=SCREAMING_SNAKE_CASE_				,							help='''An optional config json file describing the pre-trained model.'''				,							)
		lowerCamelCase :			Dict    		=parser.parse_args()

		# Extract the basename.
		lowerCamelCase :			Any    		=os.path.dirname(args.path_to_checkpoint		)

		# Load the model.
		# the .zip is very optional, let's keep it for backward compatibility
		print(F"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""		)
		if args.path_to_checkpoint.endswith('''.zip'''		):
				with zipfile.ZipFile(args.path_to_checkpoint				,							'''r'''		) as checkpoint:
						with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''		) as pytorch_dict:
								lowerCamelCase :			int    		=torch.load(SCREAMING_SNAKE_CASE_				,							map_location='''cpu'''		)
		else:
				lowerCamelCase :			str    		=torch.load(args.path_to_checkpoint				,							map_location='''cpu'''		)

		if args.config_file == "":
				# Default config of megatron-bert 345m
				lowerCamelCase :			List[str]    		=MegatronBertConfig()

				# different megatron-bert-*-345m models have different vocab sizes, so override the default
				# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
				lowerCamelCase :			Optional[int]    		=input_state_dict['''model''']['''lm_head''']['''bias'''].numel()
		else:
				lowerCamelCase :			Dict    		=MegatronBertConfig.from_json_file(args.config_file		)

		# Convert.
		print('''Converting'''		)
		lowerCamelCase :			Optional[Any]    		=convert_megatron_checkpoint(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)

		# Print the structure of converted state dict.
		if args.print_checkpoint_structure:
				recursive_print(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)

		# Store the config to file.
		print('''Saving config'''		)
		config.save_pretrained(SCREAMING_SNAKE_CASE_		)

		# Store the state_dict to file.
		lowerCamelCase :			Any    		=os.path.join(SCREAMING_SNAKE_CASE_				,							'''pytorch_model.bin'''		)
		print(F"Saving checkpoint to \"{output_checkpoint_file}\""		)
		torch.save(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)


####################################################################################################

if __name__ == "__main__":
				main()

####################################################################################################








import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTNeoConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
				import jax
				import jax.numpy as jnp

				from transformers.modeling_flax_pytorch_utils import (
				    convert_pytorch_state_dict_to_flax,
				    load_flax_weights_in_pytorch_model,
				)
				from transformers.models.gpt_neo.modeling_flax_gpt_neo import FlaxGPTNeoForCausalLM, FlaxGPTNeoModel

if is_torch_available():
				import torch
class 							snake_case_		:







		def __init__(		self      ,  __lowercase      ,  __lowercase=1_4      ,  __lowercase=7      ,  __lowercase=True      ,  __lowercase=True      ,  __lowercase=False      ,  __lowercase=True      ,  __lowercase=9_9      ,  __lowercase=3_2      ,  __lowercase=4      ,  __lowercase=4      ,  __lowercase=[[["global", "local"], 2]]      ,  __lowercase=3_7      ,  __lowercase="gelu"      ,  __lowercase=0.1      ,  __lowercase=0.1      ,  __lowercase=5_1_2      ,  __lowercase=7      ,  __lowercase=0.0_2      ,  )					-> Tuple:
				lowerCamelCase :			Optional[Any]    		=parent
				lowerCamelCase :			Optional[int]    		=batch_size
				lowerCamelCase :			Any    		=seq_length
				lowerCamelCase :			Optional[Any]    		=is_training
				lowerCamelCase :			Any    		=use_input_mask
				lowerCamelCase :			int    		=use_token_type_ids
				lowerCamelCase :			int    		=use_labels
				lowerCamelCase :			str    		=vocab_size
				lowerCamelCase :			Optional[int]    		=hidden_size
				lowerCamelCase :			int    		=num_hidden_layers
				lowerCamelCase :			List[Any]    		=num_attention_heads
				lowerCamelCase :			Any    		=attention_types
				lowerCamelCase :			List[str]    		=intermediate_size
				lowerCamelCase :			Dict    		=hidden_act
				lowerCamelCase :			Union[str, Any]    		=hidden_dropout_prob
				lowerCamelCase :			Optional[int]    		=attention_probs_dropout_prob
				lowerCamelCase :			Any    		=max_position_embeddings
				lowerCamelCase :			int    		=window_size
				lowerCamelCase :			List[Any]    		=initializer_range
				lowerCamelCase :			Any    		=None
				lowerCamelCase :			List[Any]    		=vocab_size - 1
				lowerCamelCase :			str    		=vocab_size - 1
				lowerCamelCase :			int    		=vocab_size - 1







		def       __lowercase (		self )					-> Dict:
				lowerCamelCase :			List[Any]    		=ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size )

				lowerCamelCase :			Tuple    		=None
				if self.use_input_mask:
						lowerCamelCase :			Union[str, Any]    		=random_attention_mask([self.batch_size, self.seq_length] )

				lowerCamelCase :			List[Any]    		=GPTNeoConfig(
				    vocab_size=self.vocab_size      ,  hidden_size=self.hidden_size      ,  num_layers=self.num_hidden_layers      ,  num_heads=self.num_attention_heads      ,  max_position_embeddings=self.max_position_embeddings      ,  use_cache=__lowercase      ,  bos_token_id=self.bos_token_id      ,  eos_token_id=self.eos_token_id      ,  pad_token_id=self.pad_token_id      ,  window_size=self.window_size      ,  attention_types=self.attention_types      ,  )

				return (config, input_ids, input_mask)







		def       __lowercase (		self )					-> Any:
				lowerCamelCase :			List[Any]    		=self.prepare_config_and_inputs()
				lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Optional[Any]    		=config_and_inputs
				lowerCamelCase :			List[str]    		={'''input_ids''': input_ids, '''attention_mask''': attention_mask}
				return config, inputs_dict







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )					-> List[Any]:
				lowerCamelCase :			Tuple    		=2_0
				lowerCamelCase :			Union[str, Any]    		=model_class_name(__lowercase )

				lowerCamelCase :			List[Any]    		=model.init_cache(input_ids.shape[0]      ,  __lowercase )
				lowerCamelCase :			str    		=jnp.ones((input_ids.shape[0], max_decoder_length)      ,  dtype='''i4''' )

				lowerCamelCase :			Optional[int]    		=jnp.broadcast_to(
				    jnp.arange(input_ids.shape[-1] - 1 )[None, :]      ,  (input_ids.shape[0], input_ids.shape[-1] - 1) )
				lowerCamelCase :			str    		=model(
				    input_ids[:, :-1]      ,  attention_mask=__lowercase      ,  past_key_values=__lowercase      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			List[str]    		=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]      ,  dtype='''i4''' )
				lowerCamelCase :			List[Any]    		=model(
				    input_ids[:, -1:]      ,  attention_mask=__lowercase      ,  past_key_values=outputs_cache.past_key_values      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			Any    		=model(__lowercase )

				lowerCamelCase :			Dict    		=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
				self.parent.assertTrue(diff < 1e-3      ,  msg=F"Max diff is {diff}" )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )					-> Dict:
				lowerCamelCase :			str    		=2_0
				lowerCamelCase :			Optional[int]    		=model_class_name(__lowercase )

				lowerCamelCase :			int    		=jnp.concatenate(
				    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1]) )]      ,  axis=-1      ,  )

				lowerCamelCase :			Union[str, Any]    		=model.init_cache(input_ids.shape[0]      ,  __lowercase )
				lowerCamelCase :			List[Any]    		=jnp.broadcast_to(
				    jnp.arange(input_ids.shape[-1] - 1 )[None, :]      ,  (input_ids.shape[0], input_ids.shape[-1] - 1) )

				lowerCamelCase :			Optional[Any]    		=model(
				    input_ids[:, :-1]      ,  attention_mask=__lowercase      ,  past_key_values=__lowercase      ,  position_ids=__lowercase      ,  )
				lowerCamelCase :			List[Any]    		=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]      ,  dtype='''i4''' )
				lowerCamelCase :			Union[str, Any]    		=model(
				    input_ids[:, -1:]      ,  past_key_values=outputs_cache.past_key_values      ,  attention_mask=__lowercase      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			Union[str, Any]    		=model(__lowercase      ,  attention_mask=__lowercase )

				lowerCamelCase :			Dict    		=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
				self.parent.assertTrue(diff < 1e-3      ,  msg=F"Max diff is {diff}" )

@require_flax
class 							snake_case_		(      _A    ,       _A    ,       unittest.TestCase):
		lowerCamelCase     :List[Any]							=  (FlaxGPTNeoModel, FlaxGPTNeoForCausalLM) if is_flax_available() else ()
		lowerCamelCase     :Union[str, Any]							=  (FlaxGPTNeoForCausalLM,) if is_flax_available() else ()







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			str    		=FlaxGPTNeoModelTester(self )







		def       __lowercase (		self )					-> Dict:
				for model_class_name in self.all_model_classes:
						lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Dict    		=self.model_tester.prepare_config_and_inputs()
						self.model_tester.check_use_cache_forward(__lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )







		def       __lowercase (		self )					-> Tuple:
				for model_class_name in self.all_model_classes:
						lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Optional[int]    		=self.model_tester.prepare_config_and_inputs()
						self.model_tester.check_use_cache_forward_with_attn_mask(
						    __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )







		@slow
		def       __lowercase (		self )					-> Optional[int]:
				lowerCamelCase :			Tuple    		=GPTaTokenizer.from_pretrained('''gpt2'''      ,  pad_token='''<|endoftext|>'''      ,  padding_side='''left''' )
				lowerCamelCase :			Any    		=tokenizer(['''Hello this is a long string''', '''Hey''']      ,  return_tensors='''np'''      ,  padding=__lowercase      ,  truncation=__lowercase )

				lowerCamelCase :			List[Any]    		=FlaxGPTNeoForCausalLM.from_pretrained('''EleutherAI/gpt-neo-125M''' )
				lowerCamelCase :			Union[str, Any]    		=False
				lowerCamelCase :			Optional[int]    		=model.config.eos_token_id

				lowerCamelCase :			str    		=jax.jit(model.generate )

				lowerCamelCase :			Optional[int]    		=jit_generate(
				    inputs['''input_ids''']      ,  attention_mask=inputs['''attention_mask''']      ,  pad_token_id=tokenizer.pad_token_id ).sequences

				lowerCamelCase :			Any    		=tokenizer.batch_decode(__lowercase      ,  skip_special_tokens=__lowercase )

				lowerCamelCase :			str    		=[
				    '''Hello this is a long string of text.\n\nI\'m trying to get the text of the''',
				    '''Hey, I\'m a little late to the party. I\'m going to''',
				]

				self.assertListEqual(__lowercase      ,  __lowercase )







		@is_pt_flax_cross_test
		def       __lowercase (		self )					-> int:
				lowerCamelCase		, lowerCamelCase :			int    		=self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						with self.subTest(model_class.__name__ ):
								# prepare inputs
								lowerCamelCase :			Optional[Any]    		=self._prepare_for_class(__lowercase      ,  __lowercase )
								lowerCamelCase :			Optional[Any]    		={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

								# load corresponding PyTorch class
								lowerCamelCase :			Any    		=model_class.__name__[4:]  # Skip the "Flax" at the beginning
								lowerCamelCase :			Optional[int]    		=getattr(__lowercase      ,  __lowercase )

								lowerCamelCase		, lowerCamelCase :			Union[str, Any]    		=pt_inputs['''input_ids'''].shape
								lowerCamelCase :			Dict    		=np.random.randint(0      ,  seq_length - 1      ,  size=(batch_size,) )
								for batch_idx, start_index in enumerate(__lowercase ):
										lowerCamelCase :			Tuple    		=0
										lowerCamelCase :			Optional[Any]    		=1
										lowerCamelCase :			Tuple    		=0
										lowerCamelCase :			Union[str, Any]    		=1
								lowerCamelCase :			Optional[int]    		=pt_model_class(__lowercase ).eval()
								lowerCamelCase :			Any    		=model_class(__lowercase      ,  dtype=jnp.floataa )

								lowerCamelCase :			Union[str, Any]    		=convert_pytorch_state_dict_to_flax(pt_model.state_dict()      ,  __lowercase )
								lowerCamelCase :			Optional[int]    		=fx_state

								with torch.no_grad():
										lowerCamelCase :			List[Any]    		=pt_model(**__lowercase ).to_tuple()

								lowerCamelCase :			List[Any]    		=fx_model(**__lowercase ).to_tuple()
								self.assertEqual(len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )

								with tempfile.TemporaryDirectory() as tmpdirname:
										pt_model.save_pretrained(__lowercase )
										lowerCamelCase :			List[str]    		=model_class.from_pretrained(__lowercase      ,  from_pt=__lowercase )

								lowerCamelCase :			List[str]    		=fx_model_loaded(**__lowercase ).to_tuple()
								self.assertEqual(
								    len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output_loaded, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output_loaded[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )







		@is_pt_flax_cross_test
		def       __lowercase (		self )					-> str:
				lowerCamelCase		, lowerCamelCase :			Dict    		=self.model_tester.prepare_config_and_inputs_for_common()
				for model_class in self.all_model_classes:
						with self.subTest(model_class.__name__ ):
								# prepare inputs
								lowerCamelCase :			Tuple    		=self._prepare_for_class(__lowercase      ,  __lowercase )
								lowerCamelCase :			Any    		={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

								# load corresponding PyTorch class
								lowerCamelCase :			Optional[Any]    		=model_class.__name__[4:]  # Skip the "Flax" at the beginning
								lowerCamelCase :			List[str]    		=getattr(__lowercase      ,  __lowercase )

								lowerCamelCase :			List[str]    		=pt_model_class(__lowercase ).eval()
								lowerCamelCase :			Union[str, Any]    		=model_class(__lowercase      ,  dtype=jnp.floataa )

								lowerCamelCase :			Dict    		=load_flax_weights_in_pytorch_model(__lowercase      ,  fx_model.params )
								lowerCamelCase		, lowerCamelCase :			Any    		=pt_inputs['''input_ids'''].shape
								lowerCamelCase :			str    		=np.random.randint(0      ,  seq_length - 1      ,  size=(batch_size,) )
								for batch_idx, start_index in enumerate(__lowercase ):
										lowerCamelCase :			List[str]    		=0
										lowerCamelCase :			Union[str, Any]    		=1
										lowerCamelCase :			Optional[int]    		=0
										lowerCamelCase :			Dict    		=1

								# make sure weights are tied in PyTorch
								pt_model.tie_weights()

								with torch.no_grad():
										lowerCamelCase :			Tuple    		=pt_model(**__lowercase ).to_tuple()

								lowerCamelCase :			Optional[Any]    		=fx_model(**__lowercase ).to_tuple()
								self.assertEqual(len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )

								with tempfile.TemporaryDirectory() as tmpdirname:
										fx_model.save_pretrained(__lowercase )
										lowerCamelCase :			Union[str, Any]    		=pt_model_class.from_pretrained(__lowercase      ,  from_flax=__lowercase )

								with torch.no_grad():
										lowerCamelCase :			str    		=pt_model_loaded(**__lowercase ).to_tuple()

								self.assertEqual(
								    len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )







		@slow
		def       __lowercase (		self )					-> Any:
				for model_class_name in self.all_model_classes:
						lowerCamelCase :			Optional[Any]    		=model_class_name.from_pretrained('''EleutherAI/gpt-neo-125M''' )
						lowerCamelCase :			Union[str, Any]    		=model(np.ones((1, 1) ) )
						self.assertIsNotNone(__lowercase )

