
'''simple docstring'''





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


UpperCAmelCase            = logging.getLogger(__name__)

class     a (					__magic_name__     ):




				def 						__snake_case				(				self						:					Tuple,     SCREAMING_SNAKE_CASE_						:					Union[str, Any],     SCREAMING_SNAKE_CASE_						:					Optional[int],     SCREAMING_SNAKE_CASE_						:					Optional[Any]=None,     SCREAMING_SNAKE_CASE_						:					Tuple=None   ):
										if current_layer == 0:
																snake_case							:  str			=     self.embedding_hidden_mapping_in(SCREAMING_SNAKE_CASE_   )
										else:
																snake_case							:  Any			=     hidden_states[0]

										snake_case							:  Tuple			=     int(self.config.num_hidden_layers / self.config.num_hidden_groups   )

										# Index of the hidden group
										snake_case							:  List[str]			=     int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)   )

										snake_case							:  Any			=     self.albert_layer_groups[group_idx](
										    SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group],     )
										snake_case							:  Dict			=     layer_group_output[0]

										return (hidden_states,)




@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''					,__magic_name__					,)
class     a (					__magic_name__     ):




				def __init__(				self						:					List[Any],     SCREAMING_SNAKE_CASE_						:					Optional[int]   ):
										super().__init__(SCREAMING_SNAKE_CASE_   )

										snake_case							:  Union[str, Any]			=     AlbertTransformerWithPabee(SCREAMING_SNAKE_CASE_   )

										self.init_weights()
										snake_case							:  str			=     0
										snake_case							:  Optional[int]			=     0
										snake_case							:  List[Any]			=     0

										snake_case							:  str			=     0




				def 						__snake_case				(				self						:					List[Any],     SCREAMING_SNAKE_CASE_						:					Optional[int]   ):
										snake_case							:  Any			=     threshold




				def 						__snake_case				(				self						:					List[str],     SCREAMING_SNAKE_CASE_						:					Dict   ):
										snake_case							:  Optional[int]			=     patience




				def 						__snake_case				(				self						:					str   ):
										snake_case							:  Union[str, Any]			=     0
										snake_case							:  Tuple			=     0




				def 						__snake_case				(				self						:					Optional[int]   ):
										snake_case							:  Tuple			=     self.inference_layers_num / self.inference_instances_num
										snake_case							:  Dict			=     (
										    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
										    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
										)
										print(SCREAMING_SNAKE_CASE_   )




				@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_   )
				def 						__snake_case				(				self						:					List[str],     SCREAMING_SNAKE_CASE_						:					List[Any]=None,     SCREAMING_SNAKE_CASE_						:					Union[str, Any]=None,     SCREAMING_SNAKE_CASE_						:					Tuple=None,     SCREAMING_SNAKE_CASE_						:					Dict=None,     SCREAMING_SNAKE_CASE_						:					Optional[int]=None,     SCREAMING_SNAKE_CASE_						:					Optional[int]=None,     SCREAMING_SNAKE_CASE_						:					Tuple=None,     SCREAMING_SNAKE_CASE_						:					List[Any]=None,     SCREAMING_SNAKE_CASE_						:					List[Any]=False,     ):

										if input_ids is not None and inputs_embeds is not None:
																raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''   )
										elif input_ids is not None:
																snake_case							:  int			=     input_ids.size()
										elif inputs_embeds is not None:
																snake_case							:  Optional[int]			=     inputs_embeds.size()[:-1]
										else:
																raise ValueError('''You have to specify either input_ids or inputs_embeds'''   )

										snake_case							:  Optional[int]			=     input_ids.device if input_ids is not None else inputs_embeds.device

										if attention_mask is None:
																snake_case							:  Optional[int]			=     torch.ones(SCREAMING_SNAKE_CASE_,     device=SCREAMING_SNAKE_CASE_   )
										if token_type_ids is None:
																snake_case							:  Dict			=     torch.zeros(SCREAMING_SNAKE_CASE_,     dtype=torch.long,     device=SCREAMING_SNAKE_CASE_   )

										snake_case							:  Optional[Any]			=     attention_mask.unsqueeze(1   ).unsqueeze(2   )
										snake_case							:  int			=     extended_attention_mask.to(dtype=self.dtype   )  # fp16 compatibility
										snake_case							:  str			=     (1.0 - extended_attention_mask) * -10000.0
										snake_case							:  str			=     self.get_head_mask(SCREAMING_SNAKE_CASE_,     self.config.num_hidden_layers   )

										snake_case							:  Optional[int]			=     self.embeddings(
										    SCREAMING_SNAKE_CASE_,     position_ids=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_,     inputs_embeds=SCREAMING_SNAKE_CASE_   )
										snake_case							:  str			=     embedding_output

										if self.training:
																snake_case							:  List[str]			=     []
																for i in range(self.config.num_hidden_layers   ):
																						snake_case							:  List[str]			=     self.encoder.adaptive_forward(
																						    SCREAMING_SNAKE_CASE_,     current_layer=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     head_mask=SCREAMING_SNAKE_CASE_,     )

																						snake_case							:  Union[str, Any]			=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
																						snake_case							:  Union[str, Any]			=     output_layers[i](output_dropout(SCREAMING_SNAKE_CASE_   )   )
																						res.append(SCREAMING_SNAKE_CASE_   )
										elif self.patience == 0:  # Use all layers for inference
																snake_case							:  int			=     self.encoder(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     head_mask=SCREAMING_SNAKE_CASE_   )
																snake_case							:  Optional[int]			=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
																snake_case							:  Union[str, Any]			=     [output_layers[self.config.num_hidden_layers - 1](SCREAMING_SNAKE_CASE_   )]
										else:
																snake_case							:  Optional[int]			=     0
																snake_case							:  List[Any]			=     None
																snake_case							:  Optional[int]			=     0
																for i in range(self.config.num_hidden_layers   ):
																						calculated_layer_num += 1
																						snake_case							:  Dict			=     self.encoder.adaptive_forward(
																						    SCREAMING_SNAKE_CASE_,     current_layer=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     head_mask=SCREAMING_SNAKE_CASE_,     )

																						snake_case							:  Any			=     self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
																						snake_case							:  Dict			=     output_layers[i](SCREAMING_SNAKE_CASE_   )
																						if regression:
																												snake_case							:  List[Any]			=     logits.detach()
																												if patient_result is not None:
																																		snake_case							:  Any			=     patient_result.detach()
																												if (patient_result is not None) and torch.abs(patient_result - labels   ) < self.regression_threshold:
																																		patient_counter += 1
																												else:
																																		snake_case							:  List[Any]			=     0
																						else:
																												snake_case							:  str			=     logits.detach().argmax(dim=1   )
																												if patient_result is not None:
																																		snake_case							:  str			=     patient_result.detach().argmax(dim=1   )
																												if (patient_result is not None) and torch.all(labels.eq(SCREAMING_SNAKE_CASE_   )   ):
																																		patient_counter += 1
																												else:
																																		snake_case							:  Any			=     0

																						snake_case							:  List[str]			=     logits
																						if patient_counter == self.patience:
																												break
																snake_case							:  Optional[int]			=     [patient_result]
																self.inference_layers_num += calculated_layer_num
																self.inference_instances_num += 1

										return res




@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''					,__magic_name__					,)
class     a (					__magic_name__     ):




				def __init__(				self						:					Optional[Any],     SCREAMING_SNAKE_CASE_						:					Union[str, Any]   ):
										super().__init__(SCREAMING_SNAKE_CASE_   )
										snake_case							:  List[str]			=     config.num_labels

										snake_case							:  str			=     AlbertModelWithPabee(SCREAMING_SNAKE_CASE_   )
										snake_case							:  Optional[Any]			=     nn.Dropout(config.classifier_dropout_prob   )
										snake_case							:  List[Any]			=     nn.ModuleList(
										    [nn.Linear(config.hidden_size,     self.config.num_labels   ) for _ in range(config.num_hidden_layers   )]   )

										self.init_weights()




				@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_   )
				def 						__snake_case				(				self						:					str,     SCREAMING_SNAKE_CASE_						:					Tuple=None,     SCREAMING_SNAKE_CASE_						:					str=None,     SCREAMING_SNAKE_CASE_						:					Optional[Any]=None,     SCREAMING_SNAKE_CASE_						:					Tuple=None,     SCREAMING_SNAKE_CASE_						:					str=None,     SCREAMING_SNAKE_CASE_						:					Tuple=None,     SCREAMING_SNAKE_CASE_						:					List[Any]=None,     ):
										snake_case							:  List[str]			=     self.albert(
										    input_ids=SCREAMING_SNAKE_CASE_,     attention_mask=SCREAMING_SNAKE_CASE_,     token_type_ids=SCREAMING_SNAKE_CASE_,     position_ids=SCREAMING_SNAKE_CASE_,     head_mask=SCREAMING_SNAKE_CASE_,     inputs_embeds=SCREAMING_SNAKE_CASE_,     output_dropout=self.dropout,     output_layers=self.classifiers,     regression=self.num_labels == 1,     )

										snake_case							:  List[str]			=     (logits[-1],)

										if labels is not None:
																snake_case							:  int			=     None
																snake_case							:  Dict			=     0
																for ix, logits_item in enumerate(SCREAMING_SNAKE_CASE_   ):
																						if self.num_labels == 1:
																												#  We are doing regression
																												snake_case							:  int			=     MSELoss()
																												snake_case							:  int			=     loss_fct(logits_item.view(-1   ),     labels.view(-1   )   )
																						else:
																												snake_case							:  List[Any]			=     CrossEntropyLoss()
																												snake_case							:  Optional[int]			=     loss_fct(logits_item.view(-1,     self.num_labels   ),     labels.view(-1   )   )
																						if total_loss is None:
																												snake_case							:  Dict			=     loss
																						else:
																												total_loss += loss * (ix + 1)
																						total_weights += ix + 1
																snake_case							:  List[str]			=     (total_loss / total_weights,) + outputs

										return outputs


"""simple docstring"""







import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCamelCase_							:							Optional[int]				 =		logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCamelCase_							:							Optional[Any]				 =		direct_transformers_import(Path(__file__).parent)


UpperCamelCase_							:							Optional[int]				 =		{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}


class       _lowercase		( __lowerCamelCase	):
				lowerCamelCase__							=							['feature_extractor', 'tokenizer']
				# Names need to be attr_class for attr in attributes
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None

				def __init__(  self							:     Any							,     *_lowercase							:     Optional[int]							,     **_lowercase							:     int  ):


										'''simple docstring'''
										# Sanitize args and kwargs
										for key in kwargs:
																if key not in self.attributes:
																						raise TypeError(f'Unexpected keyword argument {key}.'  )
										for arg, attribute_name in zip(_lowercase							,     self.attributes  ):
																if attribute_name in kwargs:
																						raise TypeError(f'Got multiple values for argument {attribute_name}.'  )
																else:
																						UpperCAmelCase__       						=		arg

										if len(_lowercase  ) != len(self.attributes  ):
																raise ValueError(
																    f'This processor requires {len(self.attributes  )} arguments: {", ".join(self.attributes  )}. Got '
																    f'{len(_lowercase  )} arguments instead.'  )

										# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
										for attribute_name, arg in kwargs.items():
																UpperCAmelCase__       						=		getattr(self							,     f'{attribute_name}_class'  )
																# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
																UpperCAmelCase__       						=		AUTO_TO_BASE_CLASS_MAPPING.get(_lowercase							,     _lowercase  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) for n in class_name if n is not None  )
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																if not isinstance(_lowercase							,     _lowercase  ):
																						raise ValueError(
																						    f'Received a {type(_lowercase  ).__name__} for argument {attribute_name}, but a {class_name} was expected.'  )

																setattr(self							,     _lowercase							,     _lowercase  )

				def __repr__(  self							:     Tuple  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[f'- {name}: {repr(getattr(self							,     _lowercase  )  )}' for name in self.attributes]
										UpperCAmelCase__       						=		"""\n""".join(_lowercase  )
										return f'{self.__class__.__name__}:\n{attributes_repr}'

				def 							A			(  self							:     Tuple							,     _lowercase							:     Dict							,     _lowercase							:     bool = False							,     **_lowercase							:     List[str]  ):


										'''simple docstring'''
										os.makedirs(_lowercase							,     exist_ok=_lowercase  )

										if push_to_hub:
																UpperCAmelCase__       						=		kwargs.pop("""commit_message"""							,     _lowercase  )
																UpperCAmelCase__       						=		kwargs.pop("""repo_id"""							,     save_directory.split(os.path.sep  )[-1]  )
																UpperCAmelCase__       						=		self._create_repo(_lowercase							,     **_lowercase  )
																UpperCAmelCase__       						=		self._get_files_timestamps(_lowercase  )
										# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
										# loaded from the Hub.
										if self._auto_class is not None:
																UpperCAmelCase__       						=		[getattr(self							,     _lowercase  ) for attribute_name in self.attributes]
																UpperCAmelCase__       						=		[(a.init_kwargs if isinstance(_lowercase							,     _lowercase  ) else a) for a in attrs]
																custom_object_save(self							,     _lowercase							,     config=_lowercase  )

										for attribute_name in self.attributes:
																UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																# Include the processor class in the attribute config so this processor can then be reloaded with the
																# `AutoProcessor` API.
																if hasattr(_lowercase							,     """_set_processor_class"""  ):
																						attribute._set_processor_class(self.__class__.__name__  )
																attribute.save_pretrained(_lowercase  )

										if self._auto_class is not None:
																# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
																for attribute_name in self.attributes:
																						UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																						if isinstance(_lowercase							,     _lowercase  ):
																												del attribute.init_kwargs["auto_map"]

										if push_to_hub:
																self._upload_modified_files(
																    _lowercase							,     _lowercase							,     _lowercase							,     commit_message=_lowercase							,     token=kwargs.get("""use_auth_token"""  )							,     )

				@classmethod
				def 							A			(  cls							:     Dict							,     _lowercase							:     Union[str, os.PathLike]							,     _lowercase							:     Optional[Union[str, os.PathLike]] = None							,     _lowercase							:     bool = False							,     _lowercase							:     bool = False							,     _lowercase							:     Optional[Union[str, bool]] = None							,     _lowercase							:     str = "main"							,     **_lowercase							:     Dict							,     ):


										'''simple docstring'''
										UpperCAmelCase__       						=		cache_dir
										UpperCAmelCase__       						=		force_download
										UpperCAmelCase__       						=		local_files_only
										UpperCAmelCase__       						=		revision

										UpperCAmelCase__       						=		kwargs.pop("""use_auth_token"""							,     _lowercase  )
										if use_auth_token is not None:
																warnings.warn(
																    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""							,     _lowercase  )
																if token is not None:
																						raise ValueError(
																						    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""  )
																UpperCAmelCase__       						=		use_auth_token

										if token is not None:
																# change to `token` in a follow-up PR
																UpperCAmelCase__       						=		token

										UpperCAmelCase__       						=		cls._get_arguments_from_pretrained(_lowercase							,     **_lowercase  )
										return cls(*_lowercase  )

				@classmethod
				def 							A			(  cls							:     List[Any]							,     _lowercase							:     Dict="AutoProcessor"  ):


										'''simple docstring'''
										if not isinstance(_lowercase							,     _lowercase  ):
																UpperCAmelCase__       						=		auto_class.__name__

										import transformers.models.auto as auto_module

										if not hasattr(_lowercase							,     _lowercase  ):
																raise ValueError(f'{auto_class} is not a valid auto class.'  )

										UpperCAmelCase__       						=		auto_class

				@classmethod
				def 							A			(  cls							:     Any							,     _lowercase							:     Tuple							,     **_lowercase							:     List[Any]  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[]
										for attribute_name in cls.attributes:
																UpperCAmelCase__       						=		getattr(cls							,     f'{attribute_name}_class'  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) if n is not None else None for n in class_name  )
																						UpperCAmelCase__       						=		kwargs.get("""use_fast"""							,     _lowercase  )
																						if use_fast and classes[1] is not None:
																												UpperCAmelCase__       						=		classes[1]
																						else:
																												UpperCAmelCase__       						=		classes[0]
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																args.append(attribute_class.from_pretrained(_lowercase							,     **_lowercase  )  )
										return args

				@property
				def 							A			(  self							:     int  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		getattr(self							,     self.attributes[0]  )
										return getattr(_lowercase							,     """model_input_names"""							,     _lowercase  )
UpperCamelCase_							:							List[Any]				 =		copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
						UpperCamelCase_							:							Dict				 =		ProcessorMixin.push_to_hub.__doc__.format(
						    object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
						)

