





'''simple docstring'''
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
    import torch.distributed.checkpoint as dist_cp
    from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
    from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
    from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


a__							:							Optional[Any] =get_logger(__name__)
def 				lowercase__   (   __lowercase						:  Optional[int]				,  __lowercase						:  List[str]				,  __lowercase						:  str				,  __lowercase						:  List[Any]				,  __lowercase						:  List[str]=0     ) ->      Any:



       """simple docstring"""






       os.makedirs(__lowercase				,  exist_ok=__lowercase     )
       with FSDP.state_dict_type(
           __lowercase				,  fsdp_plugin.state_dict_type				,  fsdp_plugin.state_dict_config				,  fsdp_plugin.optim_state_dict_config     ):
              __UpperCamelCase       =  model.state_dict()
              if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                     __UpperCamelCase       =  F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                     __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                     if accelerator.process_index == 0:
                            logger.info(F'''Saving model to {output_model_file}'''     )
                            torch.save(__lowercase				,  __lowercase     )
                            logger.info(F'''Model saved to {output_model_file}'''     )
              elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                     __UpperCamelCase       =  (
                         F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                         if model_index == 0
                         else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                     )
                     __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                     logger.info(F'''Saving model to {output_model_file}'''     )
                     torch.save(__lowercase				,  __lowercase     )
                     logger.info(F'''Model saved to {output_model_file}'''     )
              elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                     __UpperCamelCase       =  os.path.join(__lowercase				,  F'''{MODEL_NAME}_{model_index}'''     )
                     os.makedirs(__lowercase				,  exist_ok=__lowercase     )
                     logger.info(F'''Saving model to {ckpt_dir}'''     )
                     __UpperCamelCase       =  {'model': state_dict}

                     dist_cp.save_state_dict(
                         state_dict=__lowercase				,  storage_writer=dist_cp.FileSystemWriter(__lowercase     )				,  planner=DefaultSavePlanner()				,  )
                     logger.info(F'''Model saved to {ckpt_dir}'''     )
def 				lowercase__   (   __lowercase						:  List[str]				,  __lowercase						:  Optional[int]				,  __lowercase						:  Any				,  __lowercase						:  str				,  __lowercase						:  List[str]=0     ) ->      str:



       """simple docstring"""






       accelerator.wait_for_everyone()
       with FSDP.state_dict_type(
           __lowercase				,  fsdp_plugin.state_dict_type				,  fsdp_plugin.state_dict_config				,  fsdp_plugin.optim_state_dict_config     ):
              if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                     if type(__lowercase     ) != FSDP and accelerator.process_index != 0:
                            if not fsdp_plugin.sync_module_states:
                                   raise ValueError(
                                       'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
                                       'initializing FSDP object'     )
                            return
                     __UpperCamelCase       =  F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                     __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                     logger.info(F'''Loading model from {input_model_file}'''     )
                     __UpperCamelCase       =  torch.load(__lowercase     )
                     logger.info(F'''Model loaded from {input_model_file}'''     )
              elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                     __UpperCamelCase       =  (
                         F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                         if model_index == 0
                         else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                     )
                     __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                     logger.info(F'''Loading model from {input_model_file}'''     )
                     __UpperCamelCase       =  torch.load(__lowercase     )
                     logger.info(F'''Model loaded from {input_model_file}'''     )
              elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                     __UpperCamelCase       =  (
                         os.path.join(__lowercase				,  F'''{MODEL_NAME}_{model_index}'''     )
                         if F'''{MODEL_NAME}''' not in input_dir
                         else input_dir
                     )
                     logger.info(F'''Loading model from {ckpt_dir}'''     )
                     __UpperCamelCase       =  {'model': model.state_dict()}
                     dist_cp.load_state_dict(
                         state_dict=__lowercase				,  storage_reader=dist_cp.FileSystemReader(__lowercase     )				,  planner=DefaultLoadPlanner()				,  )
                     __UpperCamelCase       =  state_dict['model']
                     logger.info(F'''Model loaded from {ckpt_dir}'''     )
              model.load_state_dict(__lowercase     )
def 				lowercase__   (   __lowercase						:  Optional[Any]				,  __lowercase						:  Tuple				,  __lowercase						:  Optional[int]				,  __lowercase						:  Optional[Any]				,  __lowercase						:  List[Any]				,  __lowercase						:  Tuple=0     ) ->      Optional[int]:



       """simple docstring"""






       os.makedirs(__lowercase				,  exist_ok=__lowercase     )
       with FSDP.state_dict_type(
           __lowercase				,  fsdp_plugin.state_dict_type				,  fsdp_plugin.state_dict_config				,  fsdp_plugin.optim_state_dict_config     ):
              __UpperCamelCase       =  FSDP.optim_state_dict(__lowercase				,  __lowercase     )
              if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                     if accelerator.process_index == 0:
                            __UpperCamelCase       =  (
                                F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                            )
                            __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                            logger.info(F'''Saving Optimizer state to {output_optimizer_file}'''     )
                            torch.save(__lowercase				,  __lowercase     )
                            logger.info(F'''Optimizer state saved in {output_optimizer_file}'''     )
              else:
                     __UpperCamelCase       =  os.path.join(__lowercase				,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''     )
                     os.makedirs(__lowercase				,  exist_ok=__lowercase     )
                     logger.info(F'''Saving Optimizer state to {ckpt_dir}'''     )
                     dist_cp.save_state_dict(
                         state_dict={'optimizer': optim_state}				,  storage_writer=dist_cp.FileSystemWriter(__lowercase     )				,  planner=DefaultSavePlanner()				,  )
                     logger.info(F'''Optimizer state saved in {ckpt_dir}'''     )


def 				lowercase__   (   __lowercase						:  Tuple				,  __lowercase						:  List[str]				,  __lowercase						:  List[Any]				,  __lowercase						:  Optional[int]				,  __lowercase						:  str				,  __lowercase						:  Union[str, Any]=0     ) ->      List[Any]:



       """simple docstring"""






       accelerator.wait_for_everyone()
       with FSDP.state_dict_type(
           __lowercase				,  fsdp_plugin.state_dict_type				,  fsdp_plugin.state_dict_config				,  fsdp_plugin.optim_state_dict_config     ):
              if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                     __UpperCamelCase       =  None
                     # below check should work but currently it isn't working (mostly opytorch issue),
                     # in the meantime disabling it at the cost of excess memory usage
                     # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
                     __UpperCamelCase       =  (
                         F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                     )
                     __UpperCamelCase       =  os.path.join(__lowercase				,  __lowercase     )
                     logger.info(F'''Loading Optimizer state from {input_optimizer_file}'''     )
                     __UpperCamelCase       =  torch.load(__lowercase     )
                     logger.info(F'''Optimizer state loaded from {input_optimizer_file}'''     )
              else:
                     __UpperCamelCase       =  (
                         os.path.join(__lowercase				,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''     )
                         if F'''{OPTIMIZER_NAME}''' not in input_dir
                         else input_dir
                     )
                     logger.info(F'''Loading Optimizer from {ckpt_dir}'''     )
                     __UpperCamelCase       =  load_sharded_optimizer_state_dict(
                         model_state_dict=model.state_dict()				,  optimizer_key='optimizer'				,  storage_reader=dist_cp.FileSystemReader(__lowercase     )				,  )
                     __UpperCamelCase       =  optim_state['optimizer']
                     logger.info(F'''Optimizer loaded from {ckpt_dir}'''     )
              __UpperCamelCase       =  FSDP.optim_state_dict_to_load(__lowercase				,  __lowercase				,  __lowercase     )
              optimizer.load_state_dict(__lowercase     )








import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor






@require_tokenizers
@require_vision
class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->Tuple:




       """simple docstring"""


       a		  =      tempfile.mkdtemp()

       # fmt: off
       a		  =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''', ''',''', '''low''', '''lowest''']
       # fmt: on
       a		  =      os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']  )
       with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

       a		  =      {
           '''do_resize''': True,
           '''size''': {'''height''': 18, '''width''': 18},
           '''do_normalize''': True,
           '''image_mean''': [0.5, 0.5, 0.5],
           '''image_std''': [0.5, 0.5, 0.5],
       }
       a		  =      os.path.join(self.tmpdirname ,					__UpperCAmelCase  )
       with open(self.image_processor_file ,					'''w''' ,					encoding='''utf-8'''  ) as fp:
        json.dump(__UpperCAmelCase ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						List[Any] ,					**__UpperCAmelCase      :						List[Any]  )    ->int:




       """simple docstring"""


       return BertTokenizer.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any] ,					**__UpperCAmelCase      :						Optional[int]  )    ->Union[str, Any]:




       """simple docstring"""


       return ViTImageProcessor.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Any:




       """simple docstring"""


       shutil.rmtree(self.tmpdirname  )







      def       __lowerCAmelCase					(      self      :						str  )    ->Dict:




       """simple docstring"""


       a		  =      [np.random.randint(255 ,					size=(3, 30, 400) ,					dtype=np.uinta  )]

       a		  =      [Image.fromarray(np.moveaxis(__UpperCAmelCase ,					0 ,					-1  )  ) for x in image_inputs]

       return image_inputs







      def       __lowerCAmelCase					(      self      :						Any  )    ->Tuple:




       """simple docstring"""


       a		  =      self.get_tokenizer()
       a		  =      self.get_image_processor()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       processor.save_pretrained(self.tmpdirname  )
       a		  =      VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname  )

       self.assertEqual(processor.tokenizer.get_vocab() ,					tokenizer.get_vocab()  )
       self.assertIsInstance(processor.tokenizer ,					(BertTokenizer, BertTokenizerFast)  )

       self.assertEqual(processor.image_processor.to_json_string() ,					image_processor.to_json_string()  )
       self.assertIsInstance(processor.image_processor ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Dict:




       """simple docstring"""


       a		  =      VisionTextDualEncoderProcessor(
           tokenizer=self.get_tokenizer() ,					image_processor=self.get_image_processor()  )
       processor.save_pretrained(self.tmpdirname  )

       a		  =      self.get_tokenizer(bos_token='''(BOS)''' ,					eos_token='''(EOS)'''  )
       a		  =      self.get_image_processor(do_normalize=__UpperCAmelCase ,					padding_value=1.0  )

       a		  =      VisionTextDualEncoderProcessor.from_pretrained(
           self.tmpdirname ,					bos_token='''(BOS)''' ,					eos_token='''(EOS)''' ,					do_normalize=__UpperCAmelCase ,					padding_value=1.0  )

       self.assertEqual(processor.tokenizer.get_vocab() ,					tokenizer_add_kwargs.get_vocab()  )
       self.assertIsInstance(processor.tokenizer ,					(BertTokenizer, BertTokenizerFast)  )

       self.assertEqual(processor.image_processor.to_json_string() ,					image_processor_add_kwargs.to_json_string()  )
       self.assertIsInstance(processor.image_processor ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Dict:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      self.prepare_image_inputs()

       a		  =      image_processor(__UpperCAmelCase ,					return_tensors='''np'''  )
       a		  =      processor(images=__UpperCAmelCase ,					return_tensors='''np'''  )

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum() ,					input_processor[key].sum() ,					delta=1e-2  )







      def       __lowerCAmelCase					(      self      :						List[str]  )    ->str:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''

       a		  =      processor(text=__UpperCAmelCase  )

       a		  =      tokenizer(__UpperCAmelCase  )

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key] ,					encoded_processor[key]  )







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''
       a		  =      self.prepare_image_inputs()

       a		  =      processor(text=__UpperCAmelCase ,					images=__UpperCAmelCase  )

       self.assertListEqual(list(inputs.keys()  ) ,					['''input_ids''', '''token_type_ids''', '''attention_mask''', '''pixel_values''']  )

       # test if it raises when no input is passed
       with self.assertRaises(__UpperCAmelCase  ):
        processor()







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       a		  =      processor.batch_decode(__UpperCAmelCase  )
       a		  =      tokenizer.batch_decode(__UpperCAmelCase  )

       self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Dict:




       """simple docstring"""


       a		  =      self.get_image_processor()
       a		  =      self.get_tokenizer()

       a		  =      VisionTextDualEncoderProcessor(tokenizer=__UpperCAmelCase ,					image_processor=__UpperCAmelCase  )

       a		  =      '''lower newer'''
       a		  =      self.prepare_image_inputs()

       a		  =      processor(text=__UpperCAmelCase ,					images=__UpperCAmelCase  )

       self.assertListEqual(list(inputs.keys()  ) ,					processor.model_input_names  )


