


"""simple docstring"""






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig
def    A_ ( _lowerCAmelCase  :       Optional[int],  _lowerCAmelCase  :       List[str],  _lowerCAmelCase  :       Union[str, Any]=0							):






     """simple docstring"""

     if name is None:
          _a     						=					None
     else:
          _a     						=					'''.''' * max(0,  spaces - 2							) + '''# {:''' + str(50 - spaces							) + '''s}'''
          _a     						=					fmt.format(_lowerCAmelCase							)

     # Print and recurse (if needed).
     if isinstance(_lowerCAmelCase,  _lowerCAmelCase							):
          if msg is not None:
               print(_lowerCAmelCase							)
          for k in val.keys():
               recursive_print(_lowerCAmelCase,  val[k],  spaces + 2							)
     elif isinstance(_lowerCAmelCase,  torch.Tensor							):
          print(_lowerCAmelCase,  ''':''',  val.size()							)
     else:
          print(_lowerCAmelCase,  ''':''',  _lowerCAmelCase							)
def    A_ ( _lowerCAmelCase  :       List[str],  _lowerCAmelCase  :       Any,  _lowerCAmelCase  :       List[str],  _lowerCAmelCase  :       Tuple							):






     """simple docstring"""

     _a     						=					param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     _a     						=					(num_heads, num_splits, hidden_size) + input_shape[1:]
     _a     						=					param.view(*_lowerCAmelCase							)
     _a     						=					param.transpose(0,  1							).contiguous()
     _a     						=					param.view(*_lowerCAmelCase							)
     return param
def    A_ ( _lowerCAmelCase  :       Optional[int],  _lowerCAmelCase  :       Union[str, Any]							):






     """simple docstring"""

     _a     						=					config.n_positions
     _a     						=					config.n_layer
     _a     						=					config.vocab_size
     _a     						=					config.n_head
     _a     						=					config.n_embd // config.n_head

     _a     						=					sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
     _a     						=					{
         '''transformer.wte.weight''': word_embeddings,
         '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
         '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
         '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
     }

     _a     						=					'''model.language_model.encoder.layers.'''
     for i in range(_lowerCAmelCase							):
          _a     						=					torch.tril(torch.ones((n_positions, n_positions),  dtype=torch.bool							)							)
          _a     						=					causal_mask.view(1,  1,  _lowerCAmelCase,  _lowerCAmelCase							)
          _a     						=					causal_mask
          _a     						=					torch.tensor(-1e4,  dtype=torch.bfloataa							)

          _a     						=					sd_megatron[f'{pf}{i}.input_layernorm.weight']
          _a     						=					sd_megatron[f'{pf}{i}.input_layernorm.bias']

          _a     						=					sd_megatron[f'{pf}{i}.self_attention.query_key_value.weight']
          _a     						=					fix_query_key_value_ordering(_lowerCAmelCase,  3,  _lowerCAmelCase,  _lowerCAmelCase							)
          _a     						=					vala.transpose(0,  1							).contiguous()

          _a     						=					sd_megatron[f'{pf}{i}.self_attention.query_key_value.bias']
          _a     						=					fix_query_key_value_ordering(_lowerCAmelCase,  3,  _lowerCAmelCase,  _lowerCAmelCase							)
          _a     						=					vala

          _a     						=					sd_megatron[f'{pf}{i}.self_attention.dense.weight'].transpose(
              0,  1							)
          _a     						=					sd_megatron[f'{pf}{i}.self_attention.dense.bias']
          _a     						=					sd_megatron[f'{pf}{i}.post_attention_layernorm.weight']
          _a     						=					sd_megatron[f'{pf}{i}.post_attention_layernorm.bias']
          _a     						=					sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0,  1							)
          _a     						=					sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.bias']
          _a     						=					sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
              0,  1							)
          _a     						=					sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.bias']

     # For LM head, transformers' wants the matrix to weight embeddings.
     _a     						=					word_embeddings

     return sd_hf
def    A_ ( _lowerCAmelCase  :       Optional[int],  _lowerCAmelCase  :       Union[str, Any]							):






     """simple docstring"""

     _a     						=					6_40_00
     _a     						=					config_megatron['''encoder_seq_length''']
     _a     						=					config_megatron['''hidden_size''']
     _a     						=					config_megatron['''num_layers''']
     _a     						=					config_megatron['''num_attention_heads''']
     _a     						=					config_megatron['''ffn_hidden_size''']
     _a     						=					'''gelu'''
     _a     						=					0.1
     _a     						=					0.1
     _a     						=					0.1
     _a     						=					config_megatron['''layernorm_epsilon''']  # 1e-5
     _a     						=					config_megatron['''init_method_std''']  # 0.02
     _a     						=					config_megatron['''apply_query_key_layer_scaling''']  # True
     _a     						=					True
     _a     						=					True

     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 40_96:
          _a     						=					1  # <|endoftext|>
          _a     						=					1  # <|endoftext|>
          _a     						=					0  # <unk>
     else:
          _a     						=					2  # <s>
          _a     						=					3  # <|endoftext|>
          _a     						=					0  # <pad>

     return config_hf
def    A_ ( _lowerCAmelCase  :       Dict							):






     """simple docstring"""

     print(_lowerCAmelCase							)

     _a     						=					args.checkpoint_path
     _a     						=					args.save_path
     if isfile(_lowerCAmelCase							):
          raise FileNotFoundError(f'ERROR! could not find file {checkpoint_path}'							)

     # Load the model.
     _a     						=					torch.load(_lowerCAmelCase,  map_location='''cpu'''							)

     # Load the config.
     _a     						=					checkpoint['''hyper_parameters''']['''cfg''']
     _a     						=					GPTaConfig()
     _a     						=					copy_config(config_hf=_lowerCAmelCase,  config_megatron=_lowerCAmelCase							)
     _a     						=					['''GPT2LMHeadModel''']

     _a     						=					checkpoint['''state_dict''']

     # Convert.
     print('''Converting'''							)
     _a     						=					convert_megatron_checkpoint(_lowerCAmelCase,  _lowerCAmelCase							)

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(_lowerCAmelCase,  _lowerCAmelCase							)

     _a     						=					'''GPTSw3Tokenizer'''

     # Store the config to file.
     print('''Saving config'''							)
     config_hf.save_pretrained(_lowerCAmelCase							)

     # Store the state_dict to file.
     _a     						=					os.path.join(_lowerCAmelCase,  '''pytorch_model.bin'''							)
     print(f'Saving checkpoint to "{output_checkpoint_file}"'							)
     torch.save(_lowerCAmelCase,  _lowerCAmelCase							)


if __name__ == "__main__":
   __snake_case												=							argparse.ArgumentParser()
   parser.add_argument(
       '''--checkpoint_path''',
       type=str,
       required=True,
       help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
   )
   parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
   parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
   __snake_case												=							parser.parse_args()
   main(_args)
"""simple docstring"""


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import OwlViTImageProcessor






class  a__     ( unittest.TestCase						):


   def __init__(  self,					_UpperCAmelCase,					_UpperCAmelCase=7,					_UpperCAmelCase=3,					_UpperCAmelCase=18,					_UpperCAmelCase=30,					_UpperCAmelCase=400,					_UpperCAmelCase=True,					_UpperCAmelCase=None,					_UpperCAmelCase=True,					_UpperCAmelCase=None,					_UpperCAmelCase=True,					_UpperCAmelCase=[0.48_145_466, 0.4_578_275, 0.40_821_073],					_UpperCAmelCase=[0.26_862_954, 0.26_130_258, 0.27_577_711],					_UpperCAmelCase=True,					):
        '''simple docstring'''



        lowercase__								=				parent
        lowercase__								=				batch_size
        lowercase__								=				num_channels
        lowercase__								=				image_size
        lowercase__								=				min_resolution
        lowercase__								=				max_resolution
        lowercase__								=				do_resize
        lowercase__								=				size if size is not None else {"height": 18, "width": 18}
        lowercase__								=				do_center_crop
        lowercase__								=				crop_size if crop_size is not None else {"height": 18, "width": 18}
        lowercase__								=				do_normalize
        lowercase__								=				image_mean
        lowercase__								=				image_std
        lowercase__								=				do_convert_rgb


   def 						snake_case__						(  self	):
        '''simple docstring'''



        return {
            "do_resize": self.do_resize,
            "size": self.size,
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
            "do_normalize": self.do_normalize,
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_convert_rgb": self.do_convert_rgb,
        }






@require_torch
@require_vision
class  a__     ( _a       ,       unittest.TestCase						):
   snake_case_				=   OwlViTImageProcessor if is_vision_available() else None


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				OwlViTImageProcessingTester(self	)


   @property
   def 						snake_case__						(  self	):
        '''simple docstring'''



        return self.image_processor_tester.prepare_image_processor_dict()


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				self.image_processing_class(**self.image_processor_dict	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"do_resize"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"size"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"do_center_crop"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"center_crop"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"do_normalize"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"image_mean"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"image_std"	)	)
        self.assertTrue(hasattr(_UpperCAmelCase,					"do_convert_rgb"	)	)


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				self.image_processing_class.from_dict(self.image_processor_dict	)
        self.assertEqual(image_processor.size,					{"height": 18, "width": 18}	)
        self.assertEqual(image_processor.crop_size,					{"height": 18, "width": 18}	)

        lowercase__								=				self.image_processing_class.from_dict(self.image_processor_dict,					size=42,					crop_size=84	)
        self.assertEqual(image_processor.size,					{"height": 42, "width": 42}	)
        self.assertEqual(image_processor.crop_size,					{"height": 84, "width": 84}	)


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				self.image_processing_class(**self.image_processor_dict	)
        # create random PIL images
        lowercase__								=				prepare_image_inputs(self.image_processor_tester,					equal_resolution=_UpperCAmelCase	)

        for image in image_inputs:
             self.assertIsInstance(_UpperCAmelCase,					Image.Image	)

        # Test not batched input
        lowercase__								=				image_processing(image_inputs[0],					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)

        # Test batched
        lowercase__								=				image_processing(_UpperCAmelCase,					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				self.image_processing_class(**self.image_processor_dict	)
        # create random numpy tensors
        lowercase__								=				prepare_image_inputs(self.image_processor_tester,					equal_resolution=_UpperCAmelCase,					numpify=_UpperCAmelCase	)
        for image in image_inputs:
             self.assertIsInstance(_UpperCAmelCase,					np.ndarray	)

        # Test not batched input
        lowercase__								=				image_processing(image_inputs[0],					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)

        # Test batched
        lowercase__								=				image_processing(_UpperCAmelCase,					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)


   def 						snake_case__						(  self	):
        '''simple docstring'''



        lowercase__								=				self.image_processing_class(**self.image_processor_dict	)
        # create random PyTorch tensors
        lowercase__								=				prepare_image_inputs(self.image_processor_tester,					equal_resolution=_UpperCAmelCase,					torchify=_UpperCAmelCase	)
        for image in image_inputs:
             self.assertIsInstance(_UpperCAmelCase,					torch.Tensor	)

        # Test not batched input
        lowercase__								=				image_processing(image_inputs[0],					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)

        # Test batched
        lowercase__								=				image_processing(_UpperCAmelCase,					return_tensors="pt"	).pixel_values
        self.assertEqual(
            encoded_images.shape,					(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            ),					)




