

import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map






def        UpperCamelCase_(							_snake_case   :  Optional[int]	):




			"""simple docstring"""






			__a        =[]
			if isinstance(__lowerCAmelCase       ,     __lowerCAmelCase	):
						for v in tree.values():
									shapes.extend(_fetch_dims(__lowerCAmelCase	)	)
			elif isinstance(__lowerCAmelCase       ,     (list, tuple)	):
						for t in tree:
									shapes.extend(_fetch_dims(__lowerCAmelCase	)	)
			elif isinstance(__lowerCAmelCase       ,     torch.Tensor	):
						shapes.append(tree.shape	)
			else:
						raise ValueError('Not supported'	)

			return shapes






@torch.jit.ignore
def        UpperCamelCase_(							_snake_case   :  Any       ,     _snake_case   :  Dict	):




			"""simple docstring"""






			__a        =[]
			for d in reversed(__lowerCAmelCase	):
						idx.append(flat_idx % d	)
						__a        =flat_idx // d

			return tuple(reversed(__lowerCAmelCase	)	)






@torch.jit.ignore
def        UpperCamelCase_(							_snake_case   :  Tuple       ,     _snake_case   :  str       ,     _snake_case   :  int       ,     _snake_case   :  Optional[Any] = None       ,     _snake_case   :  str = None       ,     ):




			"""simple docstring"""






			def reduce_edge_list(_snake_case   :  str	) -> None:
						__a        =True
						for i in range(len(__lowerCAmelCase	)	):
									__a        =-1 * (i + 1)
									l[reversed_idx] &= tally
									__a        =l[reversed_idx]

			if start_edges is None:
						__a        =[s == 0 for s in start]
						reduce_edge_list(__lowerCAmelCase	)
			if end_edges is None:
						__a        =[e == (d - 1) for e, d in zip(__lowerCAmelCase       ,     __lowerCAmelCase	)]
						reduce_edge_list(__lowerCAmelCase	)

			# Base cases. Either start/end are empty and we're done, or the final,
			# one-dimensional tensor can be simply sliced
			if len(__lowerCAmelCase	) == 0:
						return [()]
			elif len(__lowerCAmelCase	) == 1:
						return [(slice(start[0]       ,     end[0] + 1	),)]

			__a        =[]
			__a        =[]

			# Dimensions common to start and end can be selected directly
			for s, e in zip(__lowerCAmelCase       ,     __lowerCAmelCase	):
						if s == e:
									path_list.append(slice(__lowerCAmelCase       ,     s + 1	)	)
						else:
									break

			__a        =tuple(__lowerCAmelCase	)
			__a        =len(__lowerCAmelCase	)

			# start == end, and we're done
			if divergence_idx == len(__lowerCAmelCase	):
						return [path]

			def upper() -> Tuple[Tuple[slice, ...], ...]:
						assert start_edges is not None
						assert end_edges is not None

						__a        =start[divergence_idx]
						return tuple(
						    path + (slice(__lowerCAmelCase       ,     sdi + 1	),) + s
						    for s in _get_minimal_slice_set(
						        start[divergence_idx + 1 :]       ,     [d - 1 for d in dims[divergence_idx + 1 :]]       ,     dims[divergence_idx + 1 :]       ,     start_edges=start_edges[divergence_idx + 1 :]       ,     end_edges=[True for _ in end_edges[divergence_idx + 1 :]]       ,     )	)

			def lower() -> Tuple[Tuple[slice, ...], ...]:
						assert start_edges is not None
						assert end_edges is not None

						__a        =end[divergence_idx]
						return tuple(
						    path + (slice(__lowerCAmelCase       ,     edi + 1	),) + s
						    for s in _get_minimal_slice_set(
						        [0 for _ in start[divergence_idx + 1 :]]       ,     end[divergence_idx + 1 :]       ,     dims[divergence_idx + 1 :]       ,     start_edges=[True for _ in start_edges[divergence_idx + 1 :]]       ,     end_edges=end_edges[divergence_idx + 1 :]       ,     )	)

			# If both start and end are at the edges of the subtree rooted at
			# divergence_idx, we can just select the whole subtree at once
			if start_edges[divergence_idx] and end_edges[divergence_idx]:
						slices.append(path + (slice(start[divergence_idx]       ,     end[divergence_idx] + 1	),)	)
			# If just start is at the edge, we can grab almost all of the subtree,
			# treating only the ragged bottom edge as an edge case
			elif start_edges[divergence_idx]:
						slices.append(path + (slice(start[divergence_idx]       ,     end[divergence_idx]	),)	)
						slices.extend(lower()	)
			# Analogous to the previous case, but the top is ragged this time
			elif end_edges[divergence_idx]:
						slices.extend(upper()	)
						slices.append(path + (slice(start[divergence_idx] + 1       ,     end[divergence_idx] + 1	),)	)
			# If both sides of the range are ragged, we need to handle both sides
			# separately. If there's contiguous meat in between them, we can index it
			# in one big chunk
			else:
						slices.extend(upper()	)
						__a        =end[divergence_idx] - start[divergence_idx]
						if middle_ground > 1:
									slices.append(path + (slice(start[divergence_idx] + 1       ,     end[divergence_idx]	),)	)
						slices.extend(lower()	)

			return slices






@torch.jit.ignore
def        UpperCamelCase_(							_snake_case   :  List[str]       ,     _snake_case   :  List[str]       ,     _snake_case   :  List[Any]       ,     _snake_case   :  List[Any]	):




			"""simple docstring"""






			__a        =t.shape[:no_batch_dims]
			__a        =list(_flat_idx_to_idx(__lowerCAmelCase       ,     __lowerCAmelCase	)	)
			# _get_minimal_slice_set is inclusive
			__a        =list(_flat_idx_to_idx(flat_end - 1       ,     __lowerCAmelCase	)	)

			# Get an ordered list of slices to perform
			__a        =_get_minimal_slice_set(
			    __lowerCAmelCase       ,     __lowerCAmelCase       ,     __lowerCAmelCase       ,     )

			__a        =[t[s] for s in slices]

			return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]	) for s in sliced_tensors]	)






def        UpperCamelCase_(							_snake_case   :  Optional[Any]       ,     _snake_case   :  Optional[Any]       ,     _snake_case   :  Union[str, Any]       ,     _snake_case   :  Tuple       ,     _snake_case   :  Tuple = False       ,     _snake_case   :  Optional[int] = None       ,     _snake_case   :  int = False       ,     ):




			"""simple docstring"""






			if not (len(__lowerCAmelCase	) > 0):
						raise ValueError('Must provide at least one input'	)

			__a        =[shape[:no_batch_dims] for shape in _fetch_dims(__lowerCAmelCase	)]
			__a        =tuple([max(__lowerCAmelCase	) for s in zip(*__lowerCAmelCase	)]	)

			def _prep_inputs(_snake_case   :  Optional[int]	) -> torch.Tensor:
						if not low_mem:
									if not sum(t.shape[:no_batch_dims]	) == no_batch_dims:
												__a        =t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
									__a        =t.reshape(-1       ,     *t.shape[no_batch_dims:]	)
						else:
									__a        =t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
						return t

			__a        =tensor_tree_map(_prep_inputs       ,     __lowerCAmelCase	)
			__a        =None
			if _out is not None:
						__a        =tensor_tree_map(lambda _snake_case	: t.view([-1] + list(t.shape[no_batch_dims:]	)	)       ,     _out	)

			__a        =1
			for d in orig_batch_dims:
						flat_batch_dim *= d

			__a        =flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

			def _select_chunk(_snake_case   :  Optional[Any]	) -> torch.Tensor:
						return t[i : i + chunk_size] if t.shape[0] != 1 else t

			__a        =0
			__a        =prepped_outputs
			for _ in range(__lowerCAmelCase	):
						# Chunk the input
						if not low_mem:
									__a        =_select_chunk
						else:
									__a        =partial(
									    _chunk_slice       ,     flat_start=__lowerCAmelCase       ,     flat_end=min(__lowerCAmelCase       ,     i + chunk_size	)       ,     no_batch_dims=len(__lowerCAmelCase	)       ,     )

						__a        =tensor_tree_map(__lowerCAmelCase       ,     __lowerCAmelCase	)

						# Run the layer on the chunk
						__a        =layer(**__lowerCAmelCase	)

						# Allocate space for the output
						if out is None:
									__a        =tensor_tree_map(lambda _snake_case	: t.new_zeros((flat_batch_dim,) + t.shape[1:]	)       ,     __lowerCAmelCase	)

						# Put the chunk in its pre-allocated space
						if isinstance(__lowerCAmelCase       ,     __lowerCAmelCase	):

									def assign(_snake_case   :  Dict       ,     _snake_case   :  Dict	) -> None:
												for k, v in da.items():
															if isinstance(__lowerCAmelCase       ,     __lowerCAmelCase	):
																		assign(__lowerCAmelCase       ,     da[k]	)
															else:
																		if _add_into_out:
																					v[i : i + chunk_size] += da[k]
																		else:
																					__a        =da[k]

									assign(__lowerCAmelCase       ,     __lowerCAmelCase	)
						elif isinstance(__lowerCAmelCase       ,     __lowerCAmelCase	):
									for xa, xa in zip(__lowerCAmelCase       ,     __lowerCAmelCase	):
												if _add_into_out:
															xa[i : i + chunk_size] += xa
												else:
															__a        =xa
						elif isinstance(__lowerCAmelCase       ,     torch.Tensor	):
									if _add_into_out:
												out[i : i + chunk_size] += output_chunk
									else:
												__a        =output_chunk
						else:
									raise ValueError('Not supported'	)

						i += chunk_size

			__a        =tensor_tree_map(lambda _snake_case	: t.view(orig_batch_dims + t.shape[1:]	)       ,     __lowerCAmelCase	)

			return out






class 	__magic_name__    :





			def __init__(		self					,  __snake_case = 512					,  )   -> Union[str, Any]:




						'''simple docstring'''
						__a        =max_chunk_size
						__a        =None
						__a        =None





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case					,  __snake_case					)   -> int:




						'''simple docstring'''
						logging.info('Tuning chunk size...'					)

						if min_chunk_size >= self.max_chunk_size:
									return min_chunk_size

						__a        =[2**l for l in range(int(math.log(self.max_chunk_size					,  2					)					) + 1					)]
						__a        =[c for c in candidates if c > min_chunk_size]
						__a        =[min_chunk_size] + candidates
						candidates[-1] += 4

						def test_chunk_size(__snake_case					) -> bool:
									try:
												with torch.no_grad():
															fn(*__snake_case					,  chunk_size=__snake_case					)
												return True
									except RuntimeError:
												return False

						__a        =0
						__a        =len(__snake_case					) - 1
						while i > min_viable_chunk_size_index:
									__a        =test_chunk_size(candidates[i]					)
									if not viable:
												__a        =(min_viable_chunk_size_index + i) // 2
									else:
												__a        =i
												__a        =(i + len(__snake_case					) - 1) // 2

						return candidates[min_viable_chunk_size_index]





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case					)   -> List[str]:




						'''simple docstring'''
						__a        =True
						for aa, aa in zip(__snake_case					,  __snake_case					):
									assert type(__snake_case					) == type(__snake_case					)
									if isinstance(__snake_case					,  (list, tuple)					):
												consistent &= self._compare_arg_caches(__snake_case					,  __snake_case					)
									elif isinstance(__snake_case					,  __snake_case					):
												__a        =[v for _, v in sorted(aa.items()					,  key=lambda __snake_case					: x[0]					)]
												__a        =[v for _, v in sorted(aa.items()					,  key=lambda __snake_case					: x[0]					)]
												consistent &= self._compare_arg_caches(__snake_case					,  __snake_case					)
									else:
												consistent &= aa == aa

						return consistent





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case					,  __snake_case					,  )   -> List[Any]:




						'''simple docstring'''
						__a        =True
						__a        =tree_map(lambda __snake_case					: a.shape if isinstance(__snake_case					,  torch.Tensor					) else a					,  __snake_case					,  __snake_case					)
						if self.cached_arg_data is not None:
									# If args have changed shape/value, we need to re-tune
									assert len(self.cached_arg_data					) == len(__snake_case					)
									__a        =self._compare_arg_caches(self.cached_arg_data					,  __snake_case					)
						else:
									# Otherwise, we can reuse the precomputed value
									__a        =False

						if not consistent:
									__a        =self._determine_favorable_chunk_size(
									    __snake_case					,  __snake_case					,  __snake_case					,  )
									__a        =arg_data

						assert self.cached_chunk_size is not None

						return self.cached_chunk_size














import unittest

import torch
from torch import nn

from accelerate.test_utils import require_cuda
from accelerate.utils.memory import find_executable_batch_size, release_memory


def __A    (      )->  Tuple:





						"""simple docstring"""



						raise RuntimeError('CUDA out of memory.'	)





class __lowerCamelCase (      nn.Module):





	"""simple docstring"""






	def __init__(							self    ):
							"""simple docstring"""







							super().__init__()
							_UpperCAmelCase									=			nn.Linear(3       ,    4    )
							_UpperCAmelCase									=			nn.BatchNormad(4    )
							_UpperCAmelCase									=			nn.Linear(4       ,    5    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							return self.lineara(self.batchnorm(self.lineara(UpperCAmelCase    )    )    )





class __lowerCamelCase (      unittest.TestCase):





	"""simple docstring"""






	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			[]

							@find_executable_batch_size(starting_batch_size=128    )
							def mock_training_loop_function(UpperCAmelCase    ):
													nonlocal batch_sizes
													batch_sizes.append(UpperCAmelCase    )
													if batch_size != 8:
																			raise_fake_out_of_memory()

							mock_training_loop_function()
							self.assertListEqual(UpperCAmelCase       ,    [128, 64, 32, 16, 8]    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			[]

							@find_executable_batch_size(starting_batch_size=128    )
							def mock_training_loop_function(UpperCAmelCase       ,    UpperCAmelCase    ):
													nonlocal batch_sizes
													batch_sizes.append(UpperCAmelCase    )
													if batch_size != 8:
																			raise_fake_out_of_memory()
													return batch_size, arga

							_UpperCAmelCase					, _UpperCAmelCase									=			mock_training_loop_function('hello'    )
							self.assertListEqual(UpperCAmelCase       ,    [128, 64, 32, 16, 8]    )
							self.assertListEqual([bs, arga]       ,    [8, 'hello']    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							@find_executable_batch_size(starting_batch_size=0    )
							def mock_training_loop_function(UpperCAmelCase    ):
													pass

							with self.assertRaises(UpperCAmelCase    ) as cm:
													mock_training_loop_function()
													self.assertIn('No executable batch size found, reached zero.'       ,    cm.exception.args[0]    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							@find_executable_batch_size(starting_batch_size=16    )
							def mock_training_loop_function(UpperCAmelCase    ):
													if batch_size > 0:
																			raise_fake_out_of_memory()
													pass

							with self.assertRaises(UpperCAmelCase    ) as cm:
													mock_training_loop_function()
													self.assertIn('No executable batch size found, reached zero.'       ,    cm.exception.args[0]    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							@find_executable_batch_size(starting_batch_size=128    )
							def mock_training_loop_function(UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
													if batch_size != 8:
																			raise raise_fake_out_of_memory()

							with self.assertRaises(UpperCAmelCase    ) as cm:
													mock_training_loop_function(128       ,    'hello'       ,    'world'    )
													self.assertIn('Batch size was passed into `f`'       ,    cm.exception.args[0]    )
													self.assertIn('`f(arg1=\'hello\', arg2=\'world\')'       ,    cm.exception.args[0]    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							@find_executable_batch_size(starting_batch_size=16    )
							def mock_training_loop_function(UpperCAmelCase    ):
													raise ValueError('Oops, we had an error!'    )

							with self.assertRaises(UpperCAmelCase    ) as cm:
													mock_training_loop_function()
													self.assertIn('Oops, we had an error!'       ,    cm.exception.args[0]    )

	@require_cuda
	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			torch.cuda.memory_allocated()
							_UpperCAmelCase									=			ModelForTest()
							model.cuda()
							self.assertGreater(torch.cuda.memory_allocated()       ,    UpperCAmelCase    )
							_UpperCAmelCase									=			release_memory(UpperCAmelCase    )
							self.assertEqual(torch.cuda.memory_allocated()       ,    UpperCAmelCase    )
