


'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__    :   int								=			logging.get_logger(__name__)

a__    :   str								=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a__    :   Union[str, Any]								=			{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

a__    :   Dict								=			{
    'openai-gpt': 5_1_2,
}







def _UpperCamelCase			(	__A       )				->						int:


      '''simple docstring'''



      UpperCamelCase__      =							text.strip()
      if not text:
            return []
      UpperCamelCase__      =							text.split()
      return tokens
class    lowercase_	(       _A  ):


    def __init__(	self    ,   a=True    ,   a=None    ,   a=True    ,   a=None    ,   a=True    ,   ):
          if never_split is None:
                UpperCamelCase__      =							[]
          UpperCamelCase__      =							do_lower_case
          UpperCamelCase__      =							set(__lowerCamelCase       )
          UpperCamelCase__      =							tokenize_chinese_chars
          UpperCamelCase__      =							strip_accents
          UpperCamelCase__      =							do_split_on_punc


    def   __a				(	self    ,   a    ,   a=None       ):
          UpperCamelCase__      =							self.never_split.union(set(__lowerCamelCase       )       ) if never_split else self.never_split
          UpperCamelCase__      =							self._clean_text(__lowerCamelCase       )

          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
                UpperCamelCase__      =							self._tokenize_chinese_chars(__lowerCamelCase       )
          # prevents treating the same character with different unicode codepoints as different characters
          UpperCamelCase__      =							unicodedata.normalize("NFC"    ,   __lowerCamelCase       )
          UpperCamelCase__      =							whitespace_tokenize(__lowerCamelCase       )
          UpperCamelCase__      =							[]
          for token in orig_tokens:
                if token not in never_split:
                      if self.do_lower_case:
                            UpperCamelCase__      =							token.lower()
                            if self.strip_accents is not False:
                                  UpperCamelCase__      =							self._run_strip_accents(__lowerCamelCase       )
                      elif self.strip_accents:
                            UpperCamelCase__      =							self._run_strip_accents(__lowerCamelCase       )
                split_tokens.extend(self._run_split_on_punc(__lowerCamelCase    ,   __lowerCamelCase       )       )

          UpperCamelCase__      =							whitespace_tokenize(" ".join(__lowerCamelCase       )       )
          return output_tokens


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							unicodedata.normalize("NFD"    ,   __lowerCamelCase       )
          UpperCamelCase__      =							[]
          for char in text:
                UpperCamelCase__      =							unicodedata.category(__lowerCamelCase       )
                if cat == "Mn":
                      continue
                output.append(__lowerCamelCase       )
          return "".join(__lowerCamelCase       )


    def   __a				(	self    ,   a    ,   a=None       ):
          if not self.do_split_on_punc or (never_split is not None and text in never_split):
                return [text]
          UpperCamelCase__      =							list(__lowerCamelCase       )
          UpperCamelCase__      =							0
          UpperCamelCase__      =							True
          UpperCamelCase__      =							[]
          while i < len(__lowerCamelCase       ):
                UpperCamelCase__      =							chars[i]
                if _is_punctuation(__lowerCamelCase       ):
                      output.append([char]       )
                      UpperCamelCase__      =							True
                else:
                      if start_new_word:
                            output.append([]       )
                      UpperCamelCase__      =							False
                      output[-1].append(__lowerCamelCase       )
                i += 1

          return ["".join(__lowerCamelCase       ) for x in output]


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							[]
          for char in text:
                UpperCamelCase__      =							ord(__lowerCamelCase       )
                if self._is_chinese_char(__lowerCamelCase       ):
                      output.append(" "       )
                      output.append(__lowerCamelCase       )
                      output.append(" "       )
                else:
                      output.append(__lowerCamelCase       )
          return "".join(__lowerCamelCase       )


    def   __a				(	self    ,   a       ):
          # This defines a "chinese character" as anything in the CJK Unicode block:
          #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
          #
          # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
          # despite its name. The modern Korean Hangul alphabet is a different block,
          # as is Japanese Hiragana and Katakana. Those alphabets are used to write
          # space-separated words, so they are not treated specially and handled
          # like the all of the other languages.
          if (
              (cp >= 0X4e_00 and cp <= 0X9f_ff)
              or (cp >= 0X34_00 and cp <= 0X4d_bf)  #
              or (cp >= 0X2_00_00 and cp <= 0X2_a6_df)  #
              or (cp >= 0X2_a7_00 and cp <= 0X2_b7_3f)  #
              or (cp >= 0X2_b7_40 and cp <= 0X2_b8_1f)  #
              or (cp >= 0X2_b8_20 and cp <= 0X2_ce_af)  #
              or (cp >= 0Xf9_00 and cp <= 0Xfa_ff)
              or (cp >= 0X2_f8_00 and cp <= 0X2_fa_1f)  #
          ):  #
                return True

          return False


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							[]
          for char in text:
                UpperCamelCase__      =							ord(__lowerCamelCase       )
                if cp == 0 or cp == 0Xff_fd or _is_control(__lowerCamelCase       ):
                      continue
                if _is_whitespace(__lowerCamelCase       ):
                      output.append(" "       )
                else:
                      output.append(__lowerCamelCase       )
          return "".join(__lowerCamelCase       )







def _UpperCamelCase			(	__A       )				->						Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__      =							set()
      UpperCamelCase__      =							word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)       )
            UpperCamelCase__      =							char
      return pairs







def _UpperCamelCase			(	__A       )				->						str:


      '''simple docstring'''



      UpperCamelCase__      =							text.replace("—" ,     "-"       )
      UpperCamelCase__      =							text.replace("–" ,     "-"       )
      UpperCamelCase__      =							text.replace("―" ,     "-"       )
      UpperCamelCase__      =							text.replace("…" ,     "..."       )
      UpperCamelCase__      =							text.replace("´" ,     "'"       )
      UpperCamelCase__      =							re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)" ,     R" \1 " ,     _lowerCamelCase       )
      UpperCamelCase__      =							re.sub(R"\s*\n\s*" ,     " \n " ,     _lowerCamelCase       )
      UpperCamelCase__      =							re.sub(R"[^\S\n]+" ,     " " ,     _lowerCamelCase       )
      return text.strip()



class    lowercase_	(       _A  ):
    __UpperCAmelCase       =      VOCAB_FILES_NAMES
    __UpperCAmelCase       =      PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase       =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase       =      ['input_ids', 'attention_mask']


    def __init__(	self    ,   a    ,   a    ,   a="<unk>"    ,   **a       ):
          super().__init__(unk_token=__lowerCamelCase    ,   **__lowerCamelCase       )

          try:
                import ftfy
                from spacy.lang.en import English

                UpperCamelCase__      =							English()
                UpperCamelCase__      =							_nlp.tokenizer
                UpperCamelCase__      =							ftfy.fix_text
          except ImportError:
                logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."       )
                UpperCamelCase__      =							BasicTokenizer(do_lower_case=__lowerCamelCase       )
                UpperCamelCase__      =							None

          with open(__lowerCamelCase    ,   encoding="utf-8"       ) as vocab_handle:
                UpperCamelCase__      =							json.load(__lowerCamelCase       )
          UpperCamelCase__      =							{v: k for k, v in self.encoder.items()}
          with open(__lowerCamelCase    ,   encoding="utf-8"       ) as merges_handle:
                UpperCamelCase__      =							merges_handle.read().split("\n"       )[1:-1]
          UpperCamelCase__      =							[tuple(merge.split()       ) for merge in merges]
          UpperCamelCase__      =							dict(zip(__lowerCamelCase    ,   range(len(__lowerCamelCase       )       )       )       )
          UpperCamelCase__      =							{}


    @property
    def   __a				(	self       ):
          return True


    @property
    def   __a				(	self       ):
          return len(self.encoder       )


    def   __a				(	self       ):
          return dict(self.encoder    ,   **self.added_tokens_encoder       )


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							tuple(token[:-1]       ) + (token[-1] + "</w>",)
          if token in self.cache:
                return self.cache[token]
          UpperCamelCase__      =							get_pairs(__lowerCamelCase       )

          if not pairs:
                return token + "</w>"

          while True:
                UpperCamelCase__      =							min(__lowerCamelCase    ,   key=lambda a       : self.bpe_ranks.get(__lowerCamelCase    ,   float("inf"       )       )       )
                if bigram not in self.bpe_ranks:
                      break
                UpperCamelCase__      =							bigram
                UpperCamelCase__      =							[]
                UpperCamelCase__      =							0
                while i < len(__lowerCamelCase       ):
                      try:
                            UpperCamelCase__      =							word.index(__lowerCamelCase    ,   __lowerCamelCase       )
                      except ValueError:
                            new_word.extend(word[i:]       )
                            break
                      else:
                            new_word.extend(word[i:j]       )
                            UpperCamelCase__      =							j

                      if word[i] == first and i < len(__lowerCamelCase       ) - 1 and word[i + 1] == second:
                            new_word.append(first + second       )
                            i += 2
                      else:
                            new_word.append(word[i]       )
                            i += 1
                UpperCamelCase__      =							tuple(__lowerCamelCase       )
                UpperCamelCase__      =							new_word
                if len(__lowerCamelCase       ) == 1:
                      break
                else:
                      UpperCamelCase__      =							get_pairs(__lowerCamelCase       )
          UpperCamelCase__      =							" ".join(__lowerCamelCase       )
          if word == "\n  </w>":
                UpperCamelCase__      =							"\n</w>"
          UpperCamelCase__      =							word
          return word


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							[]
          if self.fix_text is None:
                # Using BERT's BasicTokenizer
                UpperCamelCase__      =							self.nlp.tokenize(__lowerCamelCase       )
                for token in text:
                      split_tokens.extend(list(self.bpe(__lowerCamelCase       ).split(" "       )       )       )
          else:
                # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                UpperCamelCase__      =							self.nlp(text_standardize(self.fix_text(__lowerCamelCase       )       )       )
                for token in text:
                      split_tokens.extend(list(self.bpe(token.text.lower()       ).split(" "       )       )       )
          return split_tokens


    def   __a				(	self    ,   a       ):
          return self.encoder.get(__lowerCamelCase    ,   self.encoder.get(self.unk_token       )       )


    def   __a				(	self    ,   a       ):
          return self.decoder.get(__lowerCamelCase    ,   self.unk_token       )


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							"".join(__lowerCamelCase       ).replace("</w>"    ,   " "       ).strip()
          return out_string


    def   __a				(	self    ,   a    ,   a = None       ):
          if not os.path.isdir(__lowerCamelCase       ):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
                return
          UpperCamelCase__      =							os.path.join(
              __lowerCamelCase    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
          UpperCamelCase__      =							os.path.join(
              __lowerCamelCase    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

          with open(__lowerCamelCase    ,   "w"    ,   encoding="utf-8"       ) as f:
                f.write(json.dumps(self.encoder    ,   indent=2    ,   sort_keys=__lowerCamelCase    ,   ensure_ascii=__lowerCamelCase       ) + "\n"       )

          UpperCamelCase__      =							0
          with open(__lowerCamelCase    ,   "w"    ,   encoding="utf-8"       ) as writer:
                writer.write("#version: 0.2\n"       )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,   key=lambda a       : kv[1]       ):
                      if index != token_index:
                            logger.warning(
                                f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                " Please check that the tokenizer is not corrupted!"       )
                            UpperCamelCase__      =							token_index
                      writer.write(" ".join(__lowerCamelCase       ) + "\n"       )
                      index += 1

          return vocab_file, merge_file










from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__SCREAMING_SNAKE_CASE     =	logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     =	{
    """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json""",
    # See all MBART models at https://huggingface.co/models?filter=mbart
}

class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''
						a__ =	"mbart"
						a__ =	["past_key_values"]
						a__ =	{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}







						def __init__(						self   :      int						,						__lowerCamelCase   :      Optional[int]=5_02_65						,						__lowerCamelCase   :      str=10_24						,						__lowerCamelCase   :      str=12						,						__lowerCamelCase   :      Dict=40_96						,						__lowerCamelCase   :      str=16						,						__lowerCamelCase   :      int=12						,						__lowerCamelCase   :      Optional[Any]=40_96						,						__lowerCamelCase   :      Any=16						,						__lowerCamelCase   :      Optional[Any]=0.0						,						__lowerCamelCase   :      List[Any]=0.0						,						__lowerCamelCase   :      Dict=True						,						__lowerCamelCase   :      List[str]=True						,						__lowerCamelCase   :      Optional[int]="gelu"						,						__lowerCamelCase   :      List[Any]=10_24						,						__lowerCamelCase   :      Dict=0.1						,						__lowerCamelCase   :      Tuple=0.0						,						__lowerCamelCase   :      List[str]=0.0						,						__lowerCamelCase   :      Union[str, Any]=0.02						,						__lowerCamelCase   :      List[Any]=0.0						,						__lowerCamelCase   :      List[Any]=False						,						__lowerCamelCase   :      Optional[Any]=1						,						__lowerCamelCase   :      str=0						,						__lowerCamelCase   :      Optional[Any]=2						,						__lowerCamelCase   :      Dict=2						,						**__lowerCamelCase   :      int						,						)   ->					Union[str, Any]:
									A				:       Union[str, Any]														=    vocab_size
									A				:       List[str]														=    max_position_embeddings
									A				:       Optional[int]														=    d_model
									A				:       Any														=    encoder_ffn_dim
									A				:       Union[str, Any]														=    encoder_layers
									A				:       int														=    encoder_attention_heads
									A				:       Union[str, Any]														=    decoder_ffn_dim
									A				:       Any														=    decoder_layers
									A				:       Tuple														=    decoder_attention_heads
									A				:       Any														=    dropout
									A				:       Union[str, Any]														=    attention_dropout
									A				:       str														=    activation_dropout
									A				:       Dict														=    activation_function
									A				:       Optional[int]														=    init_std
									A				:       Optional[Any]														=    encoder_layerdrop
									A				:       Union[str, Any]														=    decoder_layerdrop
									A				:       Tuple														=    classifier_dropout
									A				:       Union[str, Any]														=    use_cache
									A				:       List[str]														=    encoder_layers
									A				:       List[str]														=    scale_embedding  # scale factor will be sqrt(d_model) if True
									super().__init__(
									    pad_token_id=__lowerCamelCase						,						bos_token_id=__lowerCamelCase						,						eos_token_id=__lowerCamelCase						,						is_encoder_decoder=__lowerCamelCase						,						forced_eos_token_id=__lowerCamelCase						,						**__lowerCamelCase						,						)







class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''







						@property
						def       SCREAMING_SNAKE_CASE__		(						self   :      Any			)   ->					Mapping[str, Mapping[int, str]]:
									if self.task in ["default", "seq2seq-lm"]:
												A				:       List[str]														=    OrderedDict(
												    [
												        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
												        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
												    ]			)

												if self.use_past:
															A				:       str														=    {0: "batch"}
															A				:       Optional[int]														=    {0: "batch", 1: "past_decoder_sequence + sequence"}
												else:
															A				:       Optional[int]														=    {0: "batch", 1: "decoder_sequence"}
															A				:       Union[str, Any]														=    {0: "batch", 1: "decoder_sequence"}

												if self.use_past:
															self.fill_with_past_key_values_(__lowerCamelCase						,						direction="inputs"			)
									elif self.task == "causal-lm":
												# TODO: figure this case out.
												A				:       Dict														=    OrderedDict(
												    [
												        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
												        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
												    ]			)
												if self.use_past:
															A							,			A				:       Union[str, Any]														=    self.num_layers
															for i in range(__lowerCamelCase			):
																		A				:       List[str]														=    {0: "batch", 2: "past_sequence + sequence"}
																		A				:       int														=    {0: "batch", 2: "past_sequence + sequence"}
									else:
												A				:       Optional[int]														=    OrderedDict(
												    [
												        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
												        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
												        ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
												        ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
												    ]			)

									return common_inputs







						@property
						def       SCREAMING_SNAKE_CASE__		(						self   :      int			)   ->					Mapping[str, Mapping[int, str]]:
									if self.task in ["default", "seq2seq-lm"]:
												A				:       str														=    super().outputs
									else:
												A				:       int														=    super(__lowerCamelCase						,						self			).outputs
												if self.use_past:
															A							,			A				:       Optional[int]														=    self.num_layers
															for i in range(__lowerCamelCase			):
																		A				:       List[Any]														=    {0: "batch", 2: "past_sequence + sequence"}
																		A				:       Tuple														=    {0: "batch", 2: "past_sequence + sequence"}
									return common_outputs







						def       SCREAMING_SNAKE_CASE__		(						self   :      int						,						__lowerCamelCase   :      PreTrainedTokenizer						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      bool = False						,						__lowerCamelCase   :      Optional[TensorType] = None						,						)   ->					Mapping[str, Any]:
									A				:       Tuple														=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)

									# Generate decoder inputs
									A				:       Optional[int]														=    seq_length if not self.use_past else 1
									A				:       str														=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)
									A				:       Tuple														=    {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
									A				:       Union[str, Any]														=    dict(**__lowerCamelCase						,						**__lowerCamelCase			)

									if self.use_past:
												if not is_torch_available():
															raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."			)
												else:
															import torch
												A							,			A				:       List[Any]														=    common_inputs["input_ids"].shape
												A				:       Any														=    common_inputs["decoder_input_ids"].shape[1]
												A							,			A				:       Union[str, Any]														=    self.num_attention_heads
												A				:       Tuple														=    (
												    batch,
												    num_encoder_attention_heads,
												    encoder_seq_length,
												    self._config.hidden_size // num_encoder_attention_heads,
												)
												A				:       Dict														=    decoder_seq_length + 3
												A				:       Union[str, Any]														=    (
												    batch,
												    num_decoder_attention_heads,
												    decoder_past_length,
												    self._config.hidden_size // num_decoder_attention_heads,
												)

												A				:       int														=    torch.cat(
												    [common_inputs["decoder_attention_mask"], torch.ones(__lowerCamelCase						,						__lowerCamelCase			)]						,						dim=1			)

												A				:       Optional[int]														=    []
												# If the number of encoder and decoder layers are present in the model configuration, both are considered
												A							,			A				:       Dict														=    self.num_layers
												A				:       Dict														=    min(__lowerCamelCase						,						__lowerCamelCase			)
												A				:       Optional[Any]														=    max(__lowerCamelCase						,						__lowerCamelCase			) - min_num_layers
												A				:       Tuple														=    "encoder" if num_encoder_layers > num_decoder_layers else "decoder"

												for _ in range(__lowerCamelCase			):
															common_inputs["past_key_values"].append(
															    (
															        torch.zeros(__lowerCamelCase			),
															        torch.zeros(__lowerCamelCase			),
															        torch.zeros(__lowerCamelCase			),
															        torch.zeros(__lowerCamelCase			),
															    )			)
												# TODO: test this.
												A				:       List[str]														=    encoder_shape if remaining_side_name == "encoder" else decoder_shape
												for _ in range(__lowerCamelCase						,						__lowerCamelCase			):
															common_inputs["past_key_values"].append((torch.zeros(__lowerCamelCase			), torch.zeros(__lowerCamelCase			))			)
									return common_inputs







						def       SCREAMING_SNAKE_CASE__		(						self   :      str						,						__lowerCamelCase   :      PreTrainedTokenizer						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      bool = False						,						__lowerCamelCase   :      Optional[TensorType] = None						,						)   ->					Mapping[str, Any]:
									A				:       List[Any]														=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
									    __lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)

									if self.use_past:
												if not is_torch_available():
															raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."			)
												else:
															import torch
												A							,			A				:       List[Any]														=    common_inputs["input_ids"].shape
												# Not using the same length for past_key_values
												A				:       Any														=    seqlen + 2
												A							,			A				:       List[Any]														=    self.num_layers
												A							,			A				:       int														=    self.num_attention_heads
												A				:       int														=    (
												    batch,
												    num_encoder_attention_heads,
												    past_key_values_length,
												    self._config.hidden_size // num_encoder_attention_heads,
												)

												A				:       Tuple														=    common_inputs["attention_mask"].dtype
												A				:       Dict														=    torch.cat(
												    [common_inputs["attention_mask"], torch.ones(__lowerCamelCase						,						__lowerCamelCase						,						dtype=__lowerCamelCase			)]						,						dim=1			)
												A				:       Optional[Any]														=    [
												    (torch.zeros(__lowerCamelCase			), torch.zeros(__lowerCamelCase			)) for _ in range(__lowerCamelCase			)
												]
									return common_inputs







						def       SCREAMING_SNAKE_CASE__		(						self   :      Union[str, Any]						,						__lowerCamelCase   :      PreTrainedTokenizer						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      bool = False						,						__lowerCamelCase   :      Optional[TensorType] = None						,						)   ->					Mapping[str, Any]:
									# Copied from OnnxConfig.generate_dummy_inputs
									# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
									# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
									A				:       Any														=    compute_effective_axis_dimension(
									    __lowerCamelCase						,						fixed_dimension=OnnxConfig.default_fixed_batch						,						num_token_to_add=0			)

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									A				:       Any														=    tokenizer.num_special_tokens_to_add(__lowerCamelCase			)
									A				:       Tuple														=    compute_effective_axis_dimension(
									    __lowerCamelCase						,						fixed_dimension=OnnxConfig.default_fixed_sequence						,						num_token_to_add=__lowerCamelCase			)

									# Generate dummy inputs according to compute batch and sequence
									A				:       Dict														=    [" ".join([tokenizer.unk_token]			) * seq_length] * batch_size
									A				:       Tuple														=    dict(tokenizer(__lowerCamelCase						,						return_tensors=__lowerCamelCase			)			)
									return common_inputs







						def       SCREAMING_SNAKE_CASE__		(						self   :      Union[str, Any]						,						__lowerCamelCase   :      PreTrainedTokenizer						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      int = -1						,						__lowerCamelCase   :      bool = False						,						__lowerCamelCase   :      Optional[TensorType] = None						,						)   ->					Mapping[str, Any]:
									if self.task in ["default", "seq2seq-lm"]:
												A				:       List[Any]														=    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
												    __lowerCamelCase						,						batch_size=__lowerCamelCase						,						seq_length=__lowerCamelCase						,						is_pair=__lowerCamelCase						,						framework=__lowerCamelCase			)

									elif self.task == "causal-lm":
												A				:       Tuple														=    self._generate_dummy_inputs_for_causal_lm(
												    __lowerCamelCase						,						batch_size=__lowerCamelCase						,						seq_length=__lowerCamelCase						,						is_pair=__lowerCamelCase						,						framework=__lowerCamelCase			)
									else:
												A				:       List[Any]														=    self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
												    __lowerCamelCase						,						batch_size=__lowerCamelCase						,						seq_length=__lowerCamelCase						,						is_pair=__lowerCamelCase						,						framework=__lowerCamelCase			)

									return common_inputs







						def       SCREAMING_SNAKE_CASE__		(						self   :      Union[str, Any]						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      Union[str, Any]						,						__lowerCamelCase   :      Any						,						__lowerCamelCase   :      List[Any]			)   ->					Dict:
									if self.task in ["default", "seq2seq-lm"]:
												A				:       Any														=    super()._flatten_past_key_values_(__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)
									else:
												A				:       Optional[Any]														=    super(__lowerCamelCase						,						self			)._flatten_past_key_values_(
												    __lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase						,						__lowerCamelCase			)