



import csv
import json
import logging
import os
import re
import subprocess
import tempfile
import unittest
import zipfile
from pathlib import Path
from typing import Optional
from unittest import mock

import torch

# We use TF to parse the logs
from accelerate import Accelerator
from accelerate.test_utils.testing import (
    MockingTestCase,
    TempDirTestCase,
    require_comet_ml,
    require_tensorboard,
    require_wandb,
    skip,
)
from accelerate.tracking import CometMLTracker, GeneralTracker
from accelerate.utils import ProjectConfiguration, is_comet_ml_available, is_tensorboard_available


if is_comet_ml_available():
   from comet_ml import OfflineExperiment

if is_tensorboard_available():
   import struct

   import tensorboard.compat.proto.event_pba as event_pba

__lowerCamelCase								=      logging.getLogger(__name__)


@require_tensorboard
class 	snake_case_			(unittest.TestCase	):




       """simple docstring"""






       def 			A_ (  self):

             """simple docstring"""





             UpperCAmelCase_	:						Any					   =		"test_project_with_config"
             with tempfile.TemporaryDirectory() as dirpath:
                   UpperCAmelCase_	:						List[Any]					   =		Accelerator(log_with="tensorboard"  ,project_dir=lowercase)
                   UpperCAmelCase_	:						Dict					   =		{"num_iterations": 12, "learning_rate": 1E-2, "some_boolean": False, "some_string": "some_value"}
                   accelerator.init_trackers(lowercase  ,lowercase)
                   accelerator.end_training()
                   for child in Path(F"""{dirpath}/{project_name}""").glob("*/**"):
                         UpperCAmelCase_	:						Optional[Any]					   =		list(filter(lambda lowercase: x.is_file()  ,child.iterdir()))[0]
                   self.assertNotEqual(str(lowercase)  ,"")



       def 			A_ (  self):

             """simple docstring"""





             UpperCAmelCase_	:						Tuple					   =		"test_project_with_log"
             with tempfile.TemporaryDirectory() as dirpath:
                   UpperCAmelCase_	:						str					   =		Accelerator(log_with="tensorboard"  ,project_dir=lowercase)
                   accelerator.init_trackers(lowercase)
                   UpperCAmelCase_	:						Union[str, Any]					   =		{"total_loss": 0.1, "iteration": 1, "my_text": "some_value"}
                   accelerator.log(lowercase  ,step=0)
                   accelerator.end_training()
                   # Logged values are stored in the outermost-tfevents file and can be read in as a TFRecord
                   # Names are randomly generated each time
                   UpperCAmelCase_	:						Dict					   =		list(filter(lambda lowercase: x.is_file()  ,Path(F"""{dirpath}/{project_name}""").iterdir()))[0]
                   self.assertNotEqual(str(lowercase)  ,"")



       def 			A_ (  self):

             """simple docstring"""





             UpperCAmelCase_	:						Optional[int]					   =		"test_project_with_log"
             with tempfile.TemporaryDirectory() as dirpath:
                   UpperCAmelCase_	:						str					   =		Accelerator(log_with="tensorboard"  ,project_dir=lowercase)
                   accelerator.init_trackers(lowercase)
                   UpperCAmelCase_	:						Optional[Any]					   =		{"tensor": torch.tensor(1)}
                   accelerator.log(lowercase  ,step=0)
                   accelerator.end_training()
                   # Logged values are stored in the outermost-tfevents file and can be read in as a TFRecord
                   # Names are randomly generated each time
                   UpperCAmelCase_	:						Optional[Any]					   =		list(filter(lambda lowercase: x.is_file()  ,Path(F"""{dirpath}/{project_name}""").iterdir()))[0]
                   # Reading implementation based on https://github.com/pytorch/pytorch/issues/45327#issuecomment-703757685
                   with open(lowercase  ,"rb") as f:
                         UpperCAmelCase_	:						Optional[int]					   =		f.read()
                   UpperCAmelCase_	:						Optional[int]					   =		False
                   while data:
                         UpperCAmelCase_	:						List[str]					   =		struct.unpack("Q"  ,data[:8])

                         UpperCAmelCase_	:						List[Any]					   =		data[12 : 12 + int(header[0])]  # 8+4
                         UpperCAmelCase_	:						List[Any]					   =		data[12 + int(header[0]) + 4 :]
                         UpperCAmelCase_	:						Any					   =		event_pba.Event()

                         event.ParseFromString(lowercase)
                         if event.HasField("summary"):
                               for value in event.summary.value:
                                     if value.simple_value == 1.0 and value.tag == "tensor":
                                           UpperCAmelCase_	:						Tuple					   =		True
                   self.assertTrue(lowercase  ,"Converted tensor was not found in the log file!")



       def 			A_ (  self):

             """simple docstring"""





             with self.assertRaisesRegex(lowercase  ,"Logging with `tensorboard` requires a `logging_dir`"):
                   UpperCAmelCase_	:						Union[str, Any]					   =		Accelerator(log_with="tensorboard")
             with tempfile.TemporaryDirectory() as dirpath:
                   UpperCAmelCase_	:						Union[str, Any]					   =		Accelerator(log_with="tensorboard"  ,project_dir=lowercase)



       def 			A_ (  self):

             """simple docstring"""





             UpperCAmelCase_	:						Any					   =		ProjectConfiguration(total_limit=30)
             with tempfile.TemporaryDirectory() as dirpath:
                   UpperCAmelCase_	:						List[Any]					   =		Accelerator(log_with="tensorboard"  ,project_dir=lowercase  ,project_config=lowercase)


@require_wandb
@mock.patch.dict(os.environ			,			{"""WANDB_MODE""": """offline"""}	)
class 	snake_case_			(lowercase__			,			lowercase__	):




       """simple docstring"""






       def 			A_ (  self):

             """simple docstring"""





             super().setUp()
             # wandb let's us override where logs are stored to via the WANDB_DIR env var
             self.add_mocks(mock.patch.dict(os.environ  ,{"WANDB_DIR": self.tmpdir}))



       @staticmethod
       def 			A_ (  lowercase  ,lowercase  ,lowercase = True):

             """simple docstring"""





             UpperCAmelCase_	:						int					   =		RF"""{section} ([\S\s]*?)\n\n"""
             if record:
                   UpperCAmelCase_	:						Optional[Any]					   =		RF"""Record: {pattern}"""
             UpperCAmelCase_	:						Union[str, Any]					   =		re.findall(lowercase  ,lowercase)[0]
             # A config
             if section == "config" or section == "history":
                   UpperCAmelCase_	:						Dict					   =		re.findall(R"\"([a-zA-Z0-9_.,]+)"  ,lowercase)
                   return {key: val for key, val in zip(cleaned_record[0::2]  ,cleaned_record[1::2])}
             # Everything else
             else:
                   return dict(re.findall(R"(\w+): \"([^\s]+)\""  ,lowercase))



       @skip
       def 			A_ (  self):

             """simple docstring"""





             UpperCAmelCase_	:						int					   =		"test_project_with_config"
             UpperCAmelCase_	:						List[Any]					   =		Accelerator(log_with="wandb")
             UpperCAmelCase_	:						str					   =		{"num_iterations": 12, "learning_rate": 1E-2, "some_boolean": False, "some_string": "some_value"}
             UpperCAmelCase_	:						Tuple					   =		{"wandb": {"tags": ["my_tag"]}}
             accelerator.init_trackers(lowercase  ,lowercase  ,lowercase)
             UpperCAmelCase_	:						Optional[Any]					   =		{"total_loss": 0.1, "iteration": 1, "my_text": "some_value"}
             accelerator.log(lowercase  ,step=0)
             accelerator.end_training()
             # The latest offline log is stored at wandb/latest-run/*.wandb
             for child in Path(F"""{self.tmpdir}/wandb/latest-run""").glob("*"):
                   if child.is_file() and child.suffix == ".wandb":
                         UpperCAmelCase_	:						Optional[Any]					   =		subprocess.check_output(
                             ["wandb", "sync", "--view", "--verbose", str(lowercase)]  ,env=os.environ.copy()).decode("utf8"  ,"ignore")
                         break

        # Check HPS through careful parsing and cleaning
             UpperCAmelCase_	:						List[Any]					   =		self.parse_log(lowercase  ,"config")
             self.assertEqual(logged_items["num_iterations"]  ,"12")
             self.assertEqual(logged_items["learning_rate"]  ,"0.01")
             self.assertEqual(logged_items["some_boolean"]  ,"false")
             self.assertEqual(logged_items["some_string"]  ,"some_value")
             self.assertEqual(logged_items["some_string"]  ,"some_value")

             # Run tags
             UpperCAmelCase_	:						List[Any]					   =		self.parse_log(lowercase  ,"run"  ,lowercase)
             self.assertEqual(logged_items["tags"]  ,"my_tag")

             # Actual logging
             UpperCAmelCase_	:						int					   =		self.parse_log(lowercase  ,"history")
             self.assertEqual(logged_items["total_loss"]  ,"0.1")
             self.assertEqual(logged_items["iteration"]  ,"1")
             self.assertEqual(logged_items["my_text"]  ,"some_value")
             self.assertEqual(logged_items["_step"]  ,"0")







def  _snake_case			(							self       ,	__snake_case       ,	__snake_case     )  ->  Union[str, Any]:






      '''simple docstring'''







      UpperCAmelCase_	:						Tuple					   =		run_name
      UpperCAmelCase_	:						Optional[int]					   =		OfflineExperiment(project_name=__snake_case       ,	offline_directory=__snake_case     )
      logger.info(F"""Initialized offline CometML project {self.run_name}"""     )
      logger.info("Make sure to log any initial configurations with `self.store_init_configuration` before training!"     )


@require_comet_ml
@mock.patch.object(lowercase__			,			"""__init__"""			,			lowercase__	)
class 	snake_case_			(unittest.TestCase	):




       """simple docstring"""






       @staticmethod
       def 			A_ (  lowercase  ,lowercase  ,lowercase = False):

             """simple docstring"""





             for log in log_list:
                   UpperCAmelCase_	:						Tuple					   =		json.loads(lowercase)["payload"]
                   if is_param and "param" in j.keys():
                         if j["param"]["paramName"] == key:
                               return j["param"]["paramValue"]
                   if "log_other" in j.keys():
                         if j["log_other"]["key"] == key:
                               return j["log_other"]["val"]
                   if "metric" in j.keys():
                         if j["metric"]["metricName"] == key:
                               return j["metric"]["metricValue"]



       def 			A_ (  self):

             """simple docstring"""





             with tempfile.TemporaryDirectory() as d:
                   UpperCAmelCase_	:						int					   =		CometMLTracker("test_project_with_config"  ,lowercase)
                   UpperCAmelCase_	:						List[Any]					   =		Accelerator(log_with=lowercase)
                   UpperCAmelCase_	:						Any					   =		{"num_iterations": 12, "learning_rate": 1E-2, "some_boolean": False, "some_string": "some_value"}
                   accelerator.init_trackers(lowercase  ,lowercase)
                   accelerator.end_training()
                   UpperCAmelCase_	:						Dict					   =		os.listdir(lowercase)[0]  # Comet is nice, it's just a zip file here
                   # We parse the raw logs
                   UpperCAmelCase_	:						Union[str, Any]					   =		os.path.join(lowercase  ,lowercase)
                   UpperCAmelCase_	:						List[str]					   =		zipfile.ZipFile(lowercase  ,"r")
                   UpperCAmelCase_	:						str					   =		archive.open("messages.json").read().decode("utf-8")
             UpperCAmelCase_	:						int					   =		log.split("\n")[:-1]
             self.assertEqual(self.get_value_from_key(lowercase  ,"num_iterations"  ,lowercase)  ,12)
             self.assertEqual(self.get_value_from_key(lowercase  ,"learning_rate"  ,lowercase)  ,0.01)
             self.assertEqual(self.get_value_from_key(lowercase  ,"some_boolean"  ,lowercase)  ,lowercase)
             self.assertEqual(self.get_value_from_key(lowercase  ,"some_string"  ,lowercase)  ,"some_value")



       def 			A_ (  self):

             """simple docstring"""





             with tempfile.TemporaryDirectory() as d:
                   UpperCAmelCase_	:						Dict					   =		CometMLTracker("test_project_with_config"  ,lowercase)
                   UpperCAmelCase_	:						Optional[Any]					   =		Accelerator(log_with=lowercase)
                   accelerator.init_trackers(lowercase)
                   UpperCAmelCase_	:						Tuple					   =		{"total_loss": 0.1, "iteration": 1, "my_text": "some_value"}
                   accelerator.log(lowercase  ,step=0)
                   accelerator.end_training()
                   UpperCAmelCase_	:						List[str]					   =		os.listdir(lowercase)[0]  # Comet is nice, it's just a zip file here
                   # We parse the raw logs
                   UpperCAmelCase_	:						List[str]					   =		os.path.join(lowercase  ,lowercase)
                   UpperCAmelCase_	:						Any					   =		zipfile.ZipFile(lowercase  ,"r")
                   UpperCAmelCase_	:						Union[str, Any]					   =		archive.open("messages.json").read().decode("utf-8")
             UpperCAmelCase_	:						int					   =		log.split("\n")[:-1]
             self.assertEqual(self.get_value_from_key(lowercase  ,"curr_step"  ,lowercase)  ,0)
             self.assertEqual(self.get_value_from_key(lowercase  ,"total_loss")  ,0.1)
             self.assertEqual(self.get_value_from_key(lowercase  ,"iteration")  ,1)
             self.assertEqual(self.get_value_from_key(lowercase  ,"my_text")  ,"some_value")


class 	snake_case_			(lowercase__	):




       """simple docstring"""






       _lowerCamelCase           =					[
           """total_loss""",
           """iteration""",
           """my_text""",
           """learning_rate""",
           """num_iterations""",
           """some_boolean""",
           """some_string""",
       ]

       _lowerCamelCase           =					"""my_custom_tracker"""
       _lowerCamelCase           =					False



       def __init__(  self  ,lowercase):

             """simple docstring"""





             UpperCAmelCase_	:						Optional[Any]					   =		open(F"""{dir}/log.csv"""  ,"w+")
             UpperCAmelCase_	:						List[str]					   =		csv.DictWriter(self.f  ,fieldnames=self._col_names)
             self.writer.writeheader()



       @property
       def 			A_ (  self):

             """simple docstring"""





             return self.writer



       def 			A_ (  self  ,lowercase):

             """simple docstring"""





             logger.info("Call init")
             self.writer.writerow(lowercase)



       def 			A_ (  self  ,lowercase  ,lowercase):

             """simple docstring"""





             logger.info("Call log")
             self.writer.writerow(lowercase)



       def 			A_ (  self):

             """simple docstring"""





             self.f.close()


class 	snake_case_			(unittest.TestCase	):




       """simple docstring"""






       def 			A_ (  self):

             """simple docstring"""





             with tempfile.TemporaryDirectory() as d:
                   UpperCAmelCase_	:						Union[str, Any]					   =		MyCustomTracker(lowercase)
                   UpperCAmelCase_	:						Optional[int]					   =		Accelerator(log_with=lowercase)
                   UpperCAmelCase_	:						List[Any]					   =		{"num_iterations": 12, "learning_rate": 1E-2, "some_boolean": False, "some_string": "some_value"}
                   accelerator.init_trackers("Some name"  ,lowercase)
                   accelerator.end_training()
                   with open(F"""{d}/log.csv"""  ,"r") as f:
                         UpperCAmelCase_	:						List[Any]					   =		csv.DictReader(lowercase)
                         UpperCAmelCase_	:						List[str]					   =		next(lowercase)
                         UpperCAmelCase_	:						Union[str, Any]					   =		{
                             "total_loss": "",
                             "iteration": "",
                             "my_text": "",
                             "learning_rate": "0.01",
                             "num_iterations": "12",
                             "some_boolean": "False",
                             "some_string": "some_value",
                         }
                         self.assertDictEqual(lowercase  ,lowercase)



       def 			A_ (  self):

             """simple docstring"""





             with tempfile.TemporaryDirectory() as d:
                   UpperCAmelCase_	:						Dict					   =		MyCustomTracker(lowercase)
                   UpperCAmelCase_	:						Optional[int]					   =		Accelerator(log_with=lowercase)
                   accelerator.init_trackers("Some name")
                   UpperCAmelCase_	:						Any					   =		{"total_loss": 0.1, "iteration": 1, "my_text": "some_value"}
                   accelerator.log(lowercase  ,step=0)
                   accelerator.end_training()
                   with open(F"""{d}/log.csv"""  ,"r") as f:
                         UpperCAmelCase_	:						str					   =		csv.DictReader(lowercase)
                         UpperCAmelCase_	:						Union[str, Any]					   =		next(lowercase)
                         UpperCAmelCase_	:						Any					   =		{
                             "total_loss": "0.1",
                             "iteration": "1",
                             "my_text": "some_value",
                             "learning_rate": "",
                             "num_iterations": "",
                             "some_boolean": "",
                             "some_string": "",
                         }
                         self.assertDictEqual(lowercase  ,lowercase)





import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available







def  _snake_case			(							__snake_case       ,	__snake_case       ,	__snake_case       ,	__snake_case=None     )  ->  Any:






      '''simple docstring'''







      UpperCAmelCase_	:						Dict					   =		None
      # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
      if str(weight.dtype     ) == "torch.bfloat16":
            # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
            UpperCAmelCase_	:						Optional[Any]					   =		weight.view(torch.intaa     )
            UpperCAmelCase_	:						Dict					   =		"bfloat16"
      UpperCAmelCase_	:						Optional[int]					   =		weight.cpu().numpy()
      UpperCAmelCase_	:						Tuple					   =		os.path.join(__snake_case       ,	F"""{weight_name}.dat"""     )
      if index is not None:
            if dtype is None:
                  UpperCAmelCase_	:						int					   =		str(array.dtype     )
            UpperCAmelCase_	:						Any					   =		{"dtype": dtype, "shape": list(array.shape     )}
      if array.ndim == 0:
            UpperCAmelCase_	:						List[str]					   =		array[None]
      UpperCAmelCase_	:						List[Any]					   =		np.memmap(__snake_case       ,	dtype=array.dtype       ,	mode="w+"       ,	shape=array.shape     )
      UpperCAmelCase_	:						List[str]					   =		array[:]
      file_array.flush()
      return index







def  _snake_case			(							__snake_case       ,	__snake_case     )  ->  Any:






      '''simple docstring'''







      UpperCAmelCase_	:						Union[str, Any]					   =		tuple(weight_info["shape"]     )
      if shape == ():
            # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
            UpperCAmelCase_	:						Dict					   =		(1,)

      UpperCAmelCase_	:						Tuple					   =		weight_info["dtype"]
      if dtype == "bfloat16":
            # NumPy does not support bfloat16 so this was saved as a int16
            UpperCAmelCase_	:						Tuple					   =		"int16"

      UpperCAmelCase_	:						Union[str, Any]					   =		np.memmap(__snake_case       ,	dtype=__snake_case       ,	shape=__snake_case       ,	mode="r"     )

      if len(weight_info["shape"]     ) == 0:
            UpperCAmelCase_	:						Optional[Any]					   =		weight[0]
      UpperCAmelCase_	:						List[str]					   =		torch.tensor(__snake_case     )
      if weight_info["dtype"] == "bfloat16":
            UpperCAmelCase_	:						Union[str, Any]					   =		weight.view(torch.bfloataa     )

      return weight







def  _snake_case			(							__snake_case       ,	__snake_case     )  ->  List[Any]:






      '''simple docstring'''







      if index is None or len(__snake_case     ) == 0:
            # Nothing to save
            return

      UpperCAmelCase_	:						str					   =		os.path.join(__snake_case       ,	"index.json"     )
      if os.path.isfile(__snake_case     ):
            with open(__snake_case       ,	"r"       ,	encoding="utf-8"     ) as f:
                  UpperCAmelCase_	:						str					   =		json.load(__snake_case     )
      else:
            UpperCAmelCase_	:						Dict					   =		{}
      current_index.update(__snake_case     )

      with open(__snake_case       ,	"w"       ,	encoding="utf-8"     ) as f:
            json.dump(__snake_case       ,	__snake_case       ,	indent=2     )







def  _snake_case			(							__snake_case       ,	__snake_case     )  ->  Union[str, Any]:






      '''simple docstring'''







      os.makedirs(__snake_case       ,	exist_ok=__snake_case     )
      UpperCAmelCase_	:						Any					   =		{}
      for name, parameter in state_dict.items():
            UpperCAmelCase_	:						Tuple					   =		offload_weight(__snake_case       ,	__snake_case       ,	__snake_case       ,	index=__snake_case     )

      # Update index
      save_offload_index(__snake_case       ,	__snake_case     )


class 	snake_case_			(lowercase__	):




       """simple docstring"""






       def __init__(  self  ,lowercase  ,lowercase):

             """simple docstring"""





             UpperCAmelCase_	:						Optional[Any]					   =		dataset
             UpperCAmelCase_	:						Any					   =		prefix



       def __getitem__(  self  ,lowercase):

             """simple docstring"""





             return self.dataset[F"""{self.prefix}{key}"""]



       def __iter__(  self):

             """simple docstring"""





             return iter([key for key in self.dataset if key.startswith(self.prefix)])



       def __len__(  self):

             """simple docstring"""





             return len(self.dataset)


class 	snake_case_			(lowercase__	):




       """simple docstring"""






       def __init__(  self  ,lowercase = None  ,lowercase = None  ,lowercase = None  ,lowercase=None  ,):

             """simple docstring"""





             if state_dict is None and save_folder is None:
                   raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights.")

             UpperCAmelCase_	:						Tuple					   =		{} if state_dict is None else state_dict
             UpperCAmelCase_	:						List[Any]					   =		save_folder
             if index is None and save_folder is not None:
                   with open(os.path.join(lowercase  ,"index.json")) as f:
                         UpperCAmelCase_	:						Dict					   =		json.load(lowercase)
             UpperCAmelCase_	:						List[str]					   =		{} if index is None else index
             UpperCAmelCase_	:						int					   =		list(self.state_dict.keys())
             self.all_keys.extend([key for key in self.index if key not in self.all_keys])
             UpperCAmelCase_	:						Optional[Any]					   =		device



       def __getitem__(  self  ,lowercase):

             """simple docstring"""





             if key in self.state_dict:
                   return self.state_dict[key]
             UpperCAmelCase_	:						str					   =		self.index[key]
             if weight_info.get("safetensors_file") is not None:
                   if not is_safetensors_available():
                         raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`.")

                   from safetensors import safe_open

                   UpperCAmelCase_	:						Any					   =		"cpu" if self.device is None else self.device
                   with safe_open(weight_info["safetensors_file"]  ,framework="pt"  ,device=lowercase) as f:
                         UpperCAmelCase_	:						int					   =		f.get_tensor(weight_info.get("weight_name"  ,lowercase))

                   if "dtype" in weight_info:
                         return tensor.to(getattr(lowercase  ,weight_info["dtype"]))
                   else:
                         return tensor

             UpperCAmelCase_	:						Optional[int]					   =		os.path.join(self.save_folder  ,F"""{key}.dat""")
             return load_offloaded_weight(lowercase  ,lowercase)



       def __iter__(  self):

             """simple docstring"""





             return iter(self.all_keys)



       def __len__(  self):

             """simple docstring"""





             return len(self.all_keys)







def  _snake_case			(							__snake_case       ,	__snake_case     )  ->  Optional[Any]:






      '''simple docstring'''







      UpperCAmelCase_	:						str					   =		{}
      for module_name in submodule_names:
            # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
            # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
            result.update(
                {
                    key: param
                    for key, param in state_dict.items()
                    if key == module_name or key.startswith(module_name + "."     )
                }     )
      return result
