



'''simple docstring'''


import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


lowercase											=      logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
lowercase											=      direct_transformers_import(Path(__file__).parent)


lowercase											=      {
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 __magic_name__       :			List[str]	=   ["feature_extractor", "tokenizer"]
 # Names need to be attr_class for attr in attributes
 __magic_name__       :			Dict	=   None
 __magic_name__       :			Any	=   None
 __magic_name__       :			Dict	=   None




 def __init__( self				,	*lowerCAmelCase_				,	**lowerCAmelCase_)			->   Dict:



      """simple docstring"""







      for key in kwargs:
           if key not in self.attributes:
                raise TypeError(f"""Unexpected keyword argument {key}.""")
      for arg, attribute_name in zip(lowerCAmelCase_				,	self.attributes):
           if attribute_name in kwargs:
                raise TypeError(f"""Got multiple values for argument {attribute_name}.""")
           else:
                a_									=arg

      if len(lowerCAmelCase_) != len(self.attributes):
           raise ValueError(
               f"""This processor requires {len(self.attributes)} arguments: {', '.join(self.attributes)}. Got """
               f"""{len(lowerCAmelCase_)} arguments instead.""")

      # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
      for attribute_name, arg in kwargs.items():
           a_									=getattr(self				,	f"""{attribute_name}_class""")
           # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
           a_									=AUTO_TO_BASE_CLASS_MAPPING.get(lowerCAmelCase_				,	lowerCAmelCase_)
           if isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
                a_									=tuple(getattr(lowerCAmelCase_				,	lowerCAmelCase_) for n in class_name if n is not None)
           else:
                a_									=getattr(lowerCAmelCase_				,	lowerCAmelCase_)

           if not isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
                raise ValueError(
                    f"""Received a {type(lowerCAmelCase_).__name__} for argument {attribute_name}, but a {class_name} was expected.""")

           setattr(self				,	lowerCAmelCase_				,	lowerCAmelCase_)




 def __repr__( self)			->   Optional[Any]:



      """simple docstring"""







      a_									=[f"""- {name}: {repr(getattr(self				,	lowerCAmelCase_))}""" for name in self.attributes]
      a_									="\n".join(lowerCAmelCase_)
      return f"""{self.__class__.__name__}:\n{attributes_repr}"""




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = False				,	**lowerCAmelCase_)			->   Union[str, Any]:



      """simple docstring"""







      os.makedirs(lowerCAmelCase_				,	exist_ok=lowerCAmelCase_)

      if push_to_hub:
           a_									=kwargs.pop("commit_message"				,	lowerCAmelCase_)
           a_									=kwargs.pop("repo_id"				,	save_directory.split(os.path.sep)[-1])
           a_									=self._create_repo(lowerCAmelCase_				,	**lowerCAmelCase_)
           a_									=self._get_files_timestamps(lowerCAmelCase_)
      # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
      # loaded from the Hub.
      if self._auto_class is not None:
           a_									=[getattr(self				,	lowerCAmelCase_) for attribute_name in self.attributes]
           a_									=[(a.init_kwargs if isinstance(lowerCAmelCase_				,	lowerCAmelCase_) else a) for a in attrs]
           custom_object_save(self				,	lowerCAmelCase_				,	config=lowerCAmelCase_)

      for attribute_name in self.attributes:
           a_									=getattr(self				,	lowerCAmelCase_)
           # Include the processor class in the attribute config so this processor can then be reloaded with the
           # `AutoProcessor` API.
           if hasattr(lowerCAmelCase_				,	"_set_processor_class"):
                attribute._set_processor_class(self.__class__.__name__)
           attribute.save_pretrained(lowerCAmelCase_)

      if self._auto_class is not None:
           # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
           for attribute_name in self.attributes:
                a_									=getattr(self				,	lowerCAmelCase_)
                if isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
                     del attribute.init_kwargs["auto_map"]

      if push_to_hub:
           self._upload_modified_files(
               lowerCAmelCase_				,	lowerCAmelCase_				,	lowerCAmelCase_				,	commit_message=lowerCAmelCase_				,	token=kwargs.get("use_auth_token")				,	)




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = "main"				,	**lowerCAmelCase_				,	)			->   Any:



      """simple docstring"""







      a_									=cache_dir
      a_									=force_download
      a_									=local_files_only
      a_									=revision

      a_									=kwargs.pop("use_auth_token"				,	lowerCAmelCase_)
      if use_auth_token is not None:
           warnings.warn(
               "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."				,	lowerCAmelCase_)
           if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
           a_									=use_auth_token

      if token is not None:
           # change to `token` in a follow-up PR
           a_									=token

      a_									=cls._get_arguments_from_pretrained(lowerCAmelCase_				,	**lowerCAmelCase_)
      return cls(*lowerCAmelCase_)




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_="AutoProcessor")			->   int:



      """simple docstring"""







      if not isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
           a_									=auto_class.__name__

      import transformers.models.auto as auto_module

      if not hasattr(lowerCAmelCase_				,	lowerCAmelCase_):
           raise ValueError(f"""{auto_class} is not a valid auto class.""")

      a_									=auto_class




 @classmethod
 def    lowercase_	( cls				,	lowerCAmelCase_				,	**lowerCAmelCase_)			->   Optional[int]:



      """simple docstring"""







      a_									=[]
      for attribute_name in cls.attributes:
           a_									=getattr(cls				,	f"""{attribute_name}_class""")
           if isinstance(lowerCAmelCase_				,	lowerCAmelCase_):
                a_									=tuple(getattr(lowerCAmelCase_				,	lowerCAmelCase_) if n is not None else None for n in class_name)
                a_									=kwargs.get("use_fast"				,	lowerCAmelCase_)
                if use_fast and classes[1] is not None:
                     a_									=classes[1]
                else:
                     a_									=classes[0]
           else:
                a_									=getattr(lowerCAmelCase_				,	lowerCAmelCase_)

           args.append(attribute_class.from_pretrained(lowerCAmelCase_				,	**lowerCAmelCase_))
      return args




 @property
 def    lowercase_	( self)			->   Any:



      """simple docstring"""







      a_									=getattr(self				,	self.attributes[0])
      return getattr(lowerCAmelCase_				,	"model_input_names"				,	lowerCAmelCase_)



lowercase											=      copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
  lowercase											=      ProcessorMixin.push_to_hub.__doc__.format(
      object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
  )







'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase											=      logging.get_logger(__name__)

lowercase											=      {
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 __magic_name__       :			str	=   "gpt2"
 __magic_name__       :			str	=   ["past_key_values"]
 __magic_name__       :			Dict	=   {
     "hidden_size": "n_embd",
     "max_position_embeddings": "n_positions",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }




 def __init__( self				,	lowerCAmelCase_=5_0_2_5_7				,	lowerCAmelCase_=1_0_2_4				,	lowerCAmelCase_=7_6_8				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=1_2				,	lowerCAmelCase_=None				,	lowerCAmelCase_="gelu_new"				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=1e-5				,	lowerCAmelCase_=0.0_2				,	lowerCAmelCase_="cls_index"				,	lowerCAmelCase_=True				,	lowerCAmelCase_=None				,	lowerCAmelCase_=True				,	lowerCAmelCase_=0.1				,	lowerCAmelCase_=True				,	lowerCAmelCase_=True				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=5_0_2_5_6				,	lowerCAmelCase_=False				,	lowerCAmelCase_=False				,	**lowerCAmelCase_				,	)			->   Dict:



      """simple docstring"""







      a_									=vocab_size
      a_									=n_positions
      a_									=n_embd
      a_									=n_layer
      a_									=n_head
      a_									=n_inner
      a_									=activation_function
      a_									=resid_pdrop
      a_									=embd_pdrop
      a_									=attn_pdrop
      a_									=layer_norm_epsilon
      a_									=initializer_range
      a_									=summary_type
      a_									=summary_use_proj
      a_									=summary_activation
      a_									=summary_first_dropout
      a_									=summary_proj_to_labels
      a_									=scale_attn_weights
      a_									=use_cache
      a_									=scale_attn_by_inverse_layer_idx
      a_									=reorder_and_upcast_attn

      a_									=bos_token_id
      a_									=eos_token_id

      super().__init__(bos_token_id=lowerCAmelCase_				,	eos_token_id=lowerCAmelCase_				,	**lowerCAmelCase_)







class 						UpperCAmelCase						(		__a):


 '''simple docstring'''




 def __init__( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = "default"				,	lowerCAmelCase_ = None				,	lowerCAmelCase_ = False				,	)			->   List[str]:



      """simple docstring"""







      super().__init__(lowerCAmelCase_				,	task=lowerCAmelCase_				,	patching_specs=lowerCAmelCase_				,	use_past=lowerCAmelCase_)
      if not getattr(self._config				,	"pad_token_id"				,	lowerCAmelCase_):
           # TODO: how to do that better?
           a_									=0




 @property
 def    lowercase_	( self)			->   Mapping[str, Mapping[int, str]]:



      """simple docstring"""







      a_									=OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
      if self.use_past:
           self.fill_with_past_key_values_(lowerCAmelCase_				,	direction="inputs")
           a_									={0: "batch", 1: "past_sequence + sequence"}
      else:
           a_									={0: "batch", 1: "sequence"}

      return common_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_layer




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return self._config.n_head




 def    lowercase_	( self				,	lowerCAmelCase_				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = -1				,	lowerCAmelCase_ = False				,	lowerCAmelCase_ = None				,	)			->   Mapping[str, Any]:



      """simple docstring"""







      a_									=super(lowerCAmelCase_				,	self).generate_dummy_inputs(
          lowerCAmelCase_				,	batch_size=lowerCAmelCase_				,	seq_length=lowerCAmelCase_				,	is_pair=lowerCAmelCase_				,	framework=lowerCAmelCase_)

      # We need to order the input in the way they appears in the forward()
      a_									=OrderedDict({"input_ids": common_inputs["input_ids"]})

      # Need to add the past_keys
      if self.use_past:
           if not is_torch_available():
                raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")
           else:
                import torch

                a_		,    a_									=common_inputs["input_ids"].shape
                # Not using the same length for past_key_values
                a_									=seqlen + 2
                a_									=(
                    batch,
                    self.num_attention_heads,
                    past_key_values_length,
                    self._config.hidden_size // self.num_attention_heads,
                )
                a_									=[
                    (torch.zeros(lowerCAmelCase_), torch.zeros(lowerCAmelCase_)) for _ in range(self.num_layers)
                ]

      a_									=common_inputs["attention_mask"]
      if self.use_past:
           a_									=ordered_inputs["attention_mask"].dtype
           a_									=torch.cat(
               [ordered_inputs["attention_mask"], torch.ones(lowerCAmelCase_				,	lowerCAmelCase_				,	dtype=lowerCAmelCase_)]				,	dim=1)

      return ordered_inputs




 @property
 def    lowercase_	( self)			->   int:



      """simple docstring"""







      return 1_3


