






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available







def 						_UpperCamelCase (     UpperCamelCase_							:       int     ,							UpperCamelCase_							:       List[Any]     ,							UpperCamelCase_							:       List[str]     ,							UpperCamelCase_							:       Any=None		)	-> Union[str, Any]:

						"""simple docstring"""



						lowerCAmelCase__         =    None
						# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
						if str(weight.dtype		) == "torch.bfloat16":
												# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
												lowerCAmelCase__         =    weight.view(torch.intaa		)
												lowerCAmelCase__         =    'bfloat16'
						lowerCAmelCase__         =    weight.cpu().numpy()
						lowerCAmelCase__         =    os.path.join(__UpperCamelCase     ,							F"{weight_name}.dat"		)
						if index is not None:
												if dtype is None:
																		lowerCAmelCase__         =    str(array.dtype		)
												lowerCAmelCase__         =    {'dtype': dtype, 'shape': list(array.shape		)}
						if array.ndim == 0:
												lowerCAmelCase__         =    array[None]
						lowerCAmelCase__         =    np.memmap(__UpperCamelCase     ,							dtype=array.dtype     ,							mode='w+'     ,							shape=array.shape		)
						lowerCAmelCase__         =    array[:]
						file_array.flush()
						return index







def 						_UpperCamelCase (     UpperCamelCase_							:       Union[str, Any]     ,							UpperCamelCase_							:       Any		)	-> Tuple:

						"""simple docstring"""



						lowerCAmelCase__         =    tuple(weight_info['shape']		)
						if shape == ():
												# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
												lowerCAmelCase__         =    (1,)

						lowerCAmelCase__         =    weight_info['dtype']
						if dtype == "bfloat16":
												# NumPy does not support bfloat16 so this was saved as a int16
												lowerCAmelCase__         =    'int16'

						lowerCAmelCase__         =    np.memmap(__UpperCamelCase     ,							dtype=__UpperCamelCase     ,							shape=__UpperCamelCase     ,							mode='r'		)

						if len(weight_info['shape']		) == 0:
												lowerCAmelCase__         =    weight[0]
						lowerCAmelCase__         =    torch.tensor(__UpperCamelCase		)
						if weight_info["dtype"] == "bfloat16":
												lowerCAmelCase__         =    weight.view(torch.bfloataa		)

						return weight







def 						_UpperCamelCase (     UpperCamelCase_							:       List[str]     ,							UpperCamelCase_							:       Union[str, Any]		)	-> Union[str, Any]:

						"""simple docstring"""



						if index is None or len(__UpperCamelCase		) == 0:
												# Nothing to save
												return

						lowerCAmelCase__         =    os.path.join(__UpperCamelCase     ,							'index.json'		)
						if os.path.isfile(__UpperCamelCase		):
												with open(__UpperCamelCase     ,							'r'     ,							encoding='utf-8'		) as f:
																		lowerCAmelCase__         =    json.load(__UpperCamelCase		)
						else:
												lowerCAmelCase__         =    {}
						current_index.update(__UpperCamelCase		)

						with open(__UpperCamelCase     ,							'w'     ,							encoding='utf-8'		) as f:
												json.dump(__UpperCamelCase     ,							__UpperCamelCase     ,							indent=2		)







def 						_UpperCamelCase (     UpperCamelCase_							:       Union[str, os.PathLike]     ,							UpperCamelCase_							:       Dict[str, torch.Tensor]		)	-> Tuple:

						"""simple docstring"""



						os.makedirs(__UpperCamelCase     ,							exist_ok=__UpperCamelCase		)
						lowerCAmelCase__         =    {}
						for name, parameter in state_dict.items():
												lowerCAmelCase__         =    offload_weight(__UpperCamelCase     ,							__UpperCamelCase     ,							__UpperCamelCase     ,							index=__UpperCamelCase		)

						# Update index
						save_offload_index(__UpperCamelCase     ,							__UpperCamelCase		)


class      __SCREAMING_SNAKE_CASE       ( lowerCamelCase_):






						def __init__(						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    dataset
												lowerCAmelCase__         =    prefix






						def __getitem__(						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												return self.dataset[F"{self.prefix}{key}"]






						def __iter__(						self ):






												"""simple docstring"""
												return iter([key for key in self.dataset if key.startswith(self.prefix )] )







						def __len__(						self ):






												"""simple docstring"""
												return len(self.dataset )




class      __SCREAMING_SNAKE_CASE       ( lowerCamelCase_):






						def __init__(						self  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase=None  ,    ):






												"""simple docstring"""






												if state_dict is None and save_folder is None:
																		raise ValueError('Need either a `state_dict` or a `save_folder` containing offloaded weights.' )

												lowerCAmelCase__         =    {} if state_dict is None else state_dict
												lowerCAmelCase__         =    save_folder
												if index is None and save_folder is not None:
																		with open(os.path.join(_lowercase  ,    'index.json' ) ) as f:
																								lowerCAmelCase__         =    json.load(_lowercase )
												lowerCAmelCase__         =    {} if index is None else index
												lowerCAmelCase__         =    list(self.state_dict.keys() )
												self.all_keys.extend([key for key in self.index if key not in self.all_keys] )
												lowerCAmelCase__         =    device






						def __getitem__(						self  ,    _UpperCamelCase ):






												"""simple docstring"""






												if key in self.state_dict:
																		return self.state_dict[key]
												lowerCAmelCase__         =    self.index[key]
												if weight_info.get('safetensors_file' ) is not None:
																		if not is_safetensors_available():
																								raise ImportError('These offloaded weights require the use of safetensors: `pip install safetensors`.' )

																		from safetensors import safe_open

																		lowerCAmelCase__         =    'cpu' if self.device is None else self.device
																		with safe_open(weight_info['safetensors_file']  ,    framework='pt'  ,    device=_lowercase ) as f:
																								lowerCAmelCase__         =    f.get_tensor(weight_info.get('weight_name'  ,    _lowercase ) )

																		if "dtype" in weight_info:
																								return tensor.to(getattr(_lowercase  ,    weight_info['dtype'] ) )
																		else:
																								return tensor

												lowerCAmelCase__         =    os.path.join(self.save_folder  ,    F"{key}.dat" )
												return load_offloaded_weight(_lowercase  ,    _lowercase )






						def __iter__(						self ):






												"""simple docstring"""
												return iter(self.all_keys )







						def __len__(						self ):






												"""simple docstring"""
												return len(self.all_keys )







def 						_UpperCamelCase (     UpperCamelCase_							:       Dict[str, torch.Tensor]     ,							UpperCamelCase_							:       List[str]		)	-> int:

						"""simple docstring"""



						lowerCAmelCase__         =    {}
						for module_name in submodule_names:
												# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
												# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
												result.update(
												    {
												        key: param
												        for key, param in state_dict.items()
												        if key == module_name or key.startswith(module_name + '.'		)
												    }		)
						return result






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCamelCase     :					Optional[int]				  =						{'''configuration_vilt''': ['''VILT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ViltConfig''']}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase     :					Dict				  =						['''ViltFeatureExtractor''']
   __lowerCamelCase     :					Dict				  =						['''ViltImageProcessor''']
   __lowerCamelCase     :					str				  =						['''ViltProcessor''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase     :					Any				  =						[
       '''VILT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ViltForImageAndTextRetrieval''',
       '''ViltForImagesAndTextClassification''',
       '''ViltForTokenClassification''',
       '''ViltForMaskedLM''',
       '''ViltForQuestionAnswering''',
       '''ViltLayer''',
       '''ViltModel''',
       '''ViltPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_vilt import VILT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViltConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_vilt import ViltFeatureExtractor
      from .image_processing_vilt import ViltImageProcessor
      from .processing_vilt import ViltProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_vilt import (
          VILT_PRETRAINED_MODEL_ARCHIVE_LIST,
          ViltForImageAndTextRetrieval,
          ViltForImagesAndTextClassification,
          ViltForMaskedLM,
          ViltForQuestionAnswering,
          ViltForTokenClassification,
          ViltLayer,
          ViltModel,
          ViltPreTrainedModel,
      )


else:
   import sys

   __lowerCamelCase     :					List[str]				  =						_LazyModule(__name__, globals()['''__file__'''], _import_structure)
