




import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map







def 			lowerCamelCase_		(       UpperCamelCase__				:      Union[str, Any]      )     ->       Any:





				"""simple docstring"""


				__lowerCamelCase		       =    []
				if isinstance(_lowerCamelCase ,     _lowerCamelCase      ):
								for v in tree.values():
												shapes.extend(_fetch_dims(_lowerCamelCase      )      )
				elif isinstance(_lowerCamelCase ,     (list, tuple)      ):
								for t in tree:
												shapes.extend(_fetch_dims(_lowerCamelCase      )      )
				elif isinstance(_lowerCamelCase ,     torch.Tensor      ):
								shapes.append(tree.shape      )
				else:
								raise ValueError('Not supported'      )

				return shapes







@torch.jit.ignore
def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Tuple      )     ->       str:





				"""simple docstring"""


				__lowerCamelCase		       =    []
				for d in reversed(_lowerCamelCase      ):
								idx.append(flat_idx % d      )
								__lowerCamelCase		       =    flat_idx // d

				return tuple(reversed(_lowerCamelCase      )      )







@torch.jit.ignore
def 			lowerCamelCase_		(       UpperCamelCase__				:      Optional[Any] ,     UpperCamelCase__				:      Tuple ,     UpperCamelCase__				:      Dict ,     UpperCamelCase__				:      Any = None ,     UpperCamelCase__				:      Union[str, Any] = None ,     )     ->       Optional[int]:





				"""simple docstring"""


				def reduce_edge_list(UpperCamelCase__				:      Dict      ) -> None:
								__lowerCamelCase		       =    True
								for i in range(len(_lowerCamelCase      )      ):
												__lowerCamelCase		       =    -1 * (i + 1)
												l[reversed_idx] &= tally
												__lowerCamelCase		       =    l[reversed_idx]

				if start_edges is None:
								__lowerCamelCase		       =    [s == 0 for s in start]
								reduce_edge_list(_lowerCamelCase      )
				if end_edges is None:
								__lowerCamelCase		       =    [e == (d - 1) for e, d in zip(_lowerCamelCase ,     _lowerCamelCase      )]
								reduce_edge_list(_lowerCamelCase      )

				# Base cases. Either start/end are empty and we're done, or the final,
				# one-dimensional tensor can be simply sliced
				if len(_lowerCamelCase      ) == 0:
								return [()]
				elif len(_lowerCamelCase      ) == 1:
								return [(slice(start[0] ,     end[0] + 1      ),)]

				__lowerCamelCase		       =    []
				__lowerCamelCase		       =    []

				# Dimensions common to start and end can be selected directly
				for s, e in zip(_lowerCamelCase ,     _lowerCamelCase      ):
								if s == e:
												path_list.append(slice(_lowerCamelCase ,     s + 1      )      )
								else:
												break

				__lowerCamelCase		       =    tuple(_lowerCamelCase      )
				__lowerCamelCase		       =    len(_lowerCamelCase      )

				# start == end, and we're done
				if divergence_idx == len(_lowerCamelCase      ):
								return [path]

				def upper() -> Tuple[Tuple[slice, ...], ...]:
								assert start_edges is not None
								assert end_edges is not None

								__lowerCamelCase		       =    start[divergence_idx]
								return tuple(
								    path + (slice(_lowerCamelCase ,     sdi + 1      ),) + s
								    for s in _get_minimal_slice_set(
								        start[divergence_idx + 1 :] ,     [d - 1 for d in dims[divergence_idx + 1 :]] ,     dims[divergence_idx + 1 :] ,     start_edges=start_edges[divergence_idx + 1 :] ,     end_edges=[True for _ in end_edges[divergence_idx + 1 :]] ,     )      )

				def lower() -> Tuple[Tuple[slice, ...], ...]:
								assert start_edges is not None
								assert end_edges is not None

								__lowerCamelCase		       =    end[divergence_idx]
								return tuple(
								    path + (slice(_lowerCamelCase ,     edi + 1      ),) + s
								    for s in _get_minimal_slice_set(
								        [0 for _ in start[divergence_idx + 1 :]] ,     end[divergence_idx + 1 :] ,     dims[divergence_idx + 1 :] ,     start_edges=[True for _ in start_edges[divergence_idx + 1 :]] ,     end_edges=end_edges[divergence_idx + 1 :] ,     )      )

				# If both start and end are at the edges of the subtree rooted at
				# divergence_idx, we can just select the whole subtree at once
				if start_edges[divergence_idx] and end_edges[divergence_idx]:
								slices.append(path + (slice(start[divergence_idx] ,     end[divergence_idx] + 1      ),)      )
				# If just start is at the edge, we can grab almost all of the subtree,
				# treating only the ragged bottom edge as an edge case
				elif start_edges[divergence_idx]:
								slices.append(path + (slice(start[divergence_idx] ,     end[divergence_idx]      ),)      )
								slices.extend(lower()      )
				# Analogous to the previous case, but the top is ragged this time
				elif end_edges[divergence_idx]:
								slices.extend(upper()      )
								slices.append(path + (slice(start[divergence_idx] + 1 ,     end[divergence_idx] + 1      ),)      )
				# If both sides of the range are ragged, we need to handle both sides
				# separately. If there's contiguous meat in between them, we can index it
				# in one big chunk
				else:
								slices.extend(upper()      )
								__lowerCamelCase		       =    end[divergence_idx] - start[divergence_idx]
								if middle_ground > 1:
												slices.append(path + (slice(start[divergence_idx] + 1 ,     end[divergence_idx]      ),)      )
								slices.extend(lower()      )

				return slices







@torch.jit.ignore
def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      List[Any] ,     UpperCamelCase__				:      Tuple ,     UpperCamelCase__				:      List[str]      )     ->       Tuple:





				"""simple docstring"""


				__lowerCamelCase		       =    t.shape[:no_batch_dims]
				__lowerCamelCase		       =    list(_flat_idx_to_idx(_lowerCamelCase ,     _lowerCamelCase      )      )
				# _get_minimal_slice_set is inclusive
				__lowerCamelCase		       =    list(_flat_idx_to_idx(flat_end - 1 ,     _lowerCamelCase      )      )

				# Get an ordered list of slices to perform
				__lowerCamelCase		       =    _get_minimal_slice_set(
				    _lowerCamelCase ,     _lowerCamelCase ,     _lowerCamelCase ,     )

				__lowerCamelCase		       =    [t[s] for s in slices]

				return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]      ) for s in sliced_tensors]      )







def 			lowerCamelCase_		(       UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Tuple ,     UpperCamelCase__				:      List[str] ,     UpperCamelCase__				:      Dict ,     UpperCamelCase__				:      str = False ,     UpperCamelCase__				:      Tuple = None ,     UpperCamelCase__				:      Any = False ,     )     ->       Optional[Any]:





				"""simple docstring"""


				if not (len(_lowerCamelCase      ) > 0):
								raise ValueError('Must provide at least one input'      )

				__lowerCamelCase		       =    [shape[:no_batch_dims] for shape in _fetch_dims(_lowerCamelCase      )]
				__lowerCamelCase		       =    tuple([max(_lowerCamelCase      ) for s in zip(*_lowerCamelCase      )]      )

				def _prep_inputs(UpperCamelCase__				:      str      ) -> torch.Tensor:
								if not low_mem:
												if not sum(t.shape[:no_batch_dims]      ) == no_batch_dims:
																__lowerCamelCase		       =    t.expand(orig_batch_dims + t.shape[no_batch_dims:]      )
												__lowerCamelCase		       =    t.reshape(-1 ,     *t.shape[no_batch_dims:]      )
								else:
												__lowerCamelCase		       =    t.expand(orig_batch_dims + t.shape[no_batch_dims:]      )
								return t

				__lowerCamelCase		       =    tensor_tree_map(_prep_inputs ,     _lowerCamelCase      )
				__lowerCamelCase		       =    None
				if _out is not None:
								__lowerCamelCase		       =    tensor_tree_map(lambda UpperCamelCase__      : t.view([-1] + list(t.shape[no_batch_dims:]      )      ) ,     _out      )

				__lowerCamelCase		       =    1
				for d in orig_batch_dims:
								flat_batch_dim *= d

				__lowerCamelCase		       =    flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

				def _select_chunk(UpperCamelCase__				:      List[str]      ) -> torch.Tensor:
								return t[i : i + chunk_size] if t.shape[0] != 1 else t

				__lowerCamelCase		       =    0
				__lowerCamelCase		       =    prepped_outputs
				for _ in range(_lowerCamelCase      ):
								# Chunk the input
								if not low_mem:
												__lowerCamelCase		       =    _select_chunk
								else:
												__lowerCamelCase		       =    partial(
												    _chunk_slice ,     flat_start=_lowerCamelCase ,     flat_end=min(_lowerCamelCase ,     i + chunk_size      ) ,     no_batch_dims=len(_lowerCamelCase      ) ,     )

								__lowerCamelCase		       =    tensor_tree_map(_lowerCamelCase ,     _lowerCamelCase      )

								# Run the layer on the chunk
								__lowerCamelCase		       =    layer(**_lowerCamelCase      )

								# Allocate space for the output
								if out is None:
												__lowerCamelCase		       =    tensor_tree_map(lambda UpperCamelCase__      : t.new_zeros((flat_batch_dim,) + t.shape[1:]      ) ,     _lowerCamelCase      )

								# Put the chunk in its pre-allocated space
								if isinstance(_lowerCamelCase ,     _lowerCamelCase      ):

												def assign(UpperCamelCase__				:      Any ,     UpperCamelCase__				:      List[str]      ) -> None:
																for k, v in da.items():
																				if isinstance(_lowerCamelCase ,     _lowerCamelCase      ):
																								assign(_lowerCamelCase ,     da[k]      )
																				else:
																								if _add_into_out:
																												v[i : i + chunk_size] += da[k]
																								else:
																												__lowerCamelCase		       =    da[k]

												assign(_lowerCamelCase ,     _lowerCamelCase      )
								elif isinstance(_lowerCamelCase ,     _lowerCamelCase      ):
												for xa, xa in zip(_lowerCamelCase ,     _lowerCamelCase      ):
																if _add_into_out:
																				xa[i : i + chunk_size] += xa
																else:
																				__lowerCamelCase		       =    xa
								elif isinstance(_lowerCamelCase ,     torch.Tensor      ):
												if _add_into_out:
																out[i : i + chunk_size] += output_chunk
												else:
																__lowerCamelCase		       =    output_chunk
								else:
												raise ValueError('Not supported'      )

								i += chunk_size

				__lowerCamelCase		       =    tensor_tree_map(lambda UpperCamelCase__      : t.view(orig_batch_dims + t.shape[1:]      ) ,     _lowerCamelCase      )

				return out







class     __lowerCAmelCase :

			"""simple docstring"""







			def __init__(      self			,						lowerCamelCase__ = 512			,						)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    max_chunk_size
							__lowerCamelCase		       =    None
							__lowerCamelCase		       =    None


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__			,						lowerCamelCase__							)    ->      str:



							'''simple docstring'''
							logging.info('Tuning chunk size...'							)

							if min_chunk_size >= self.max_chunk_size:
											return min_chunk_size

							__lowerCamelCase		       =    [2**l for l in range(int(math.log(self.max_chunk_size			,						2							)							) + 1							)]
							__lowerCamelCase		       =    [c for c in candidates if c > min_chunk_size]
							__lowerCamelCase		       =    [min_chunk_size] + candidates
							candidates[-1] += 4

							def test_chunk_size(lowerCamelCase__							) -> bool:
											try:
															with torch.no_grad():
																			fn(*__a			,						chunk_size=__a							)
															return True
											except RuntimeError:
															return False

							__lowerCamelCase		       =    0
							__lowerCamelCase		       =    len(__a							) - 1
							while i > min_viable_chunk_size_index:
											__lowerCamelCase		       =    test_chunk_size(candidates[i]							)
											if not viable:
															__lowerCamelCase		       =    (min_viable_chunk_size_index + i) // 2
											else:
															__lowerCamelCase		       =    i
															__lowerCamelCase		       =    (i + len(__a							) - 1) // 2

							return candidates[min_viable_chunk_size_index]


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    True
							for aa, aa in zip(__a			,						__a							):
											assert type(__a							) == type(__a							)
											if isinstance(__a			,						(list, tuple)							):
															consistent &= self._compare_arg_caches(__a			,						__a							)
											elif isinstance(__a			,						__a							):
															__lowerCamelCase		       =    [v for _, v in sorted(aa.items()			,						key=lambda lowerCamelCase__							: x[0]							)]
															__lowerCamelCase		       =    [v for _, v in sorted(aa.items()			,						key=lambda lowerCamelCase__							: x[0]							)]
															consistent &= self._compare_arg_caches(__a			,						__a							)
											else:
															consistent &= aa == aa

							return consistent






			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__			,						lowerCamelCase__			,						)    ->      Optional[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    True
							__lowerCamelCase		       =    tree_map(lambda lowerCamelCase__							: a.shape if isinstance(__a			,						torch.Tensor							) else a			,						__a			,						__a							)
							if self.cached_arg_data is not None:
											# If args have changed shape/value, we need to re-tune
											assert len(self.cached_arg_data							) == len(__a							)
											__lowerCamelCase		       =    self._compare_arg_caches(self.cached_arg_data			,						__a							)
							else:
											# Otherwise, we can reuse the precomputed value
											__lowerCamelCase		       =    False

							if not consistent:
											__lowerCamelCase		       =    self._determine_favorable_chunk_size(
											    __a			,						__a			,						__a			,						)
											__lowerCamelCase		       =    arg_data

							assert self.cached_chunk_size is not None

							return self.cached_chunk_size







from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '   ,					a   ,					)
class   UpperCAmelCase_    (       a):
							lowerCamelCase__   			=		RobertaConfig
							lowerCamelCase__   			=		'roberta'







							def __init__(   self,			__a):

														'''simple docstring'''

														super().__init__(__a)

														_lowerCAmelCase     :							Optional[Any]    =    RobertaEmbeddings(__a)
														self.init_weights()



@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '   ,					a   ,					)
class   UpperCAmelCase_    (       a):
							lowerCamelCase__   			=		RobertaConfig
							lowerCamelCase__   			=		'roberta'







							def __init__(   self,			__a):

														'''simple docstring'''

														super().__init__(__a)
														_lowerCAmelCase     :							Optional[int]    =    config.num_labels
														_lowerCAmelCase     :							Optional[int]    =    config.num_hidden_layers

														_lowerCAmelCase     :							Optional[int]    =    DeeRobertaModel(__a)
														_lowerCAmelCase     :							Union[str, Any]    =    nn.Dropout(config.hidden_dropout_prob)
														_lowerCAmelCase     :							List[str]    =    nn.Linear(config.hidden_size,			self.config.num_labels)







							@add_start_docstrings_to_model_forward(__a)
							def    snake_case__      (   self,			__a=None,			__a=None,			__a=None,			__a=None,			__a=None,			__a=None,			__a=None,			__a=-1,			__a=False,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    self.num_layers
														try:
																					_lowerCAmelCase     :							List[Any]    =    self.roberta(
																					    __a,			attention_mask=__a,			token_type_ids=__a,			position_ids=__a,			head_mask=__a,			inputs_embeds=__a,			)

																					_lowerCAmelCase     :							List[Any]    =    outputs[1]

																					_lowerCAmelCase     :							Dict    =    self.dropout(__a)
																					_lowerCAmelCase     :							Dict    =    self.classifier(__a)
																					_lowerCAmelCase     :							Optional[Any]    =    (logits,) + outputs[2:]  # add hidden states and attention if they are here
														except HighwayException as e:
																					_lowerCAmelCase     :							Tuple    =    e.message
																					_lowerCAmelCase     :							Union[str, Any]    =    e.exit_layer
																					_lowerCAmelCase     :							List[Any]    =    outputs[0]

														if not self.training:
																					_lowerCAmelCase     :							int    =    entropy(__a)
																					_lowerCAmelCase     :							List[Any]    =    []
																					_lowerCAmelCase     :							str    =    []
														if labels is not None:
																					if self.num_labels == 1:
																												#  We are doing regression
																												_lowerCAmelCase     :							Optional[Any]    =    MSELoss()
																												_lowerCAmelCase     :							int    =    loss_fct(logits.view(-1),			labels.view(-1))
																					else:
																												_lowerCAmelCase     :							Optional[Any]    =    CrossEntropyLoss()
																												_lowerCAmelCase     :							Optional[Any]    =    loss_fct(logits.view(-1,			self.num_labels),			labels.view(-1))

																					# work with highway exits
																					_lowerCAmelCase     :							Optional[int]    =    []
																					for highway_exit in outputs[-1]:
																												_lowerCAmelCase     :							Any    =    highway_exit[0]
																												if not self.training:
																																			highway_logits_all.append(__a)
																																			highway_entropy.append(highway_exit[2])
																												if self.num_labels == 1:
																																			#  We are doing regression
																																			_lowerCAmelCase     :							List[str]    =    MSELoss()
																																			_lowerCAmelCase     :							List[Any]    =    loss_fct(highway_logits.view(-1),			labels.view(-1))
																												else:
																																			_lowerCAmelCase     :							Dict    =    CrossEntropyLoss()
																																			_lowerCAmelCase     :							Optional[Any]    =    loss_fct(highway_logits.view(-1,			self.num_labels),			labels.view(-1))
																												highway_losses.append(__a)

																					if train_highway:
																												_lowerCAmelCase     :							int    =    (sum(highway_losses[:-1]),) + outputs
																												# exclude the final highway, of course
																					else:
																												_lowerCAmelCase     :							Any    =    (loss,) + outputs
														if not self.training:
																					_lowerCAmelCase     :							Optional[Any]    =    outputs + ((original_entropy, highway_entropy), exit_layer)
																					if output_layer >= 0:
																												_lowerCAmelCase     :							Optional[Any]    =    (
																												    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																												)  # use the highway of the last layer

														return outputs  # (loss), logits, (hidden_states), (attentions), entropy
