





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase_     							= logging.get_logger(__name__)

UpperCAmelCase_     							= {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_     							= {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_     							= {
    'Salesforce/codegen-350M-mono': 2048,
}






class     lowercase__       (	_lowerCamelCase				):

   '''simple docstring'''


   a						:							Dict   		=		VOCAB_FILES_NAMES
   a						:							Optional[Any]   		=		PRETRAINED_VOCAB_FILES_MAP
   a						:							str   		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a						:							List[str]   		=		["input_ids", "attention_mask"]
   a						:							Dict   		=		CodeGenTokenizer




   def __init__(    self,		__magic_name__=None,		__magic_name__=None,		__magic_name__=None,		__magic_name__="<|endoftext|>",		__magic_name__="<|endoftext|>",		__magic_name__="<|endoftext|>",		__magic_name__=False,		**__magic_name__,		) ->      Any:






       """simple docstring"""
       super().__init__(
           lowercase_,		lowercase_,		tokenizer_file=lowercase_,		unk_token=lowercase_,		bos_token=lowercase_,		eos_token=lowercase_,		add_prefix_space=lowercase_,		**lowercase_,		)

       if kwargs.pop('''add_bos_token''',		lowercase_							):
           UpperCamelCase__							:				Tuple          =      kwargs.pop('''name_or_path''',		''''''							)
           raise ValueError(
               '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
               '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
               f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
               f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
               '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
               ''' so that the fast tokenizer works correctly.'''							)

       UpperCamelCase__							:				Any          =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',		lowercase_							) != add_prefix_space:
           UpperCamelCase__							:				str          =      getattr(lowercase_,		pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:				Union[str, Any]          =      add_prefix_space
           UpperCamelCase__							:				Tuple          =      pre_tok_class(**lowercase_							)

       UpperCamelCase__							:				int          =      add_prefix_space




   def       UpperCamelCase__  (    self,		*__magic_name__,		**__magic_name__							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      kwargs.get('''is_split_into_words''',		lowercase_							)
       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*lowercase_,		**lowercase_							)




   def       UpperCamelCase__  (    self,		*__magic_name__,		**__magic_name__							) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				List[str]          =      kwargs.get('''is_split_into_words''',		lowercase_							)

       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*lowercase_,		**lowercase_							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__ = None							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				Dict          =      self._tokenizer.model.save(lowercase_,		name=lowercase_							)
       return tuple(lowercase_							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__ = False,		__magic_name__ = None,		__magic_name__ = None,		**__magic_name__,		) ->      Any:






       """simple docstring"""
       UpperCamelCase__							:				int          =      super().decode(
           token_ids=lowercase_,		skip_special_tokens=lowercase_,		clean_up_tokenization_spaces=lowercase_,		**lowercase_,		)

       if truncate_before_pattern is not None and len(lowercase_							) > 0:
           UpperCamelCase__							:				str          =      self.truncate(lowercase_,		lowercase_							)

       return decoded_text




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__							) ->      int:






       """simple docstring"""
       def find_re(__magic_name__,		__magic_name__,		__magic_name__							):
           UpperCamelCase__							:				Optional[int]          =      pattern.search(lowercase_,		lowercase_							)
           return m.start() if m else -1

       UpperCamelCase__							:				List[Any]          =      [re.compile(lowercase_,		re.MULTILINE							) for pattern in truncate_before_pattern]

       UpperCamelCase__							:				List[str]          =      list(re.finditer('''^print''',		lowercase_,		re.MULTILINE							)							)

       if len(lowercase_							) > 1:
           UpperCamelCase__							:				List[str]          =      completion[: prints[1].start()]

       UpperCamelCase__							:				Optional[int]          =      list(re.finditer('''^def''',		lowercase_,		re.MULTILINE							)							)

       if len(lowercase_							) > 1:
           UpperCamelCase__							:				Union[str, Any]          =      completion[: defs[1].start()]

       UpperCamelCase__							:				Optional[Any]          =      0

       UpperCamelCase__							:				Union[str, Any]          =      [
           pos for pos in [find_re(lowercase_,		lowercase_,		lowercase_							) for terminal in terminals] if pos != -1
       ]

       if len(lowercase_							) > 0:
           return completion[: min(lowercase_							)]
       else:
           return completion







'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a  :				Dict									=  logging.get_logger(__name__)

a  :				List[str]									=  {
    'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/config.json',
    # See all Marian models at https://huggingface.co/models?filter=marian
}






class 						a				(   _lowerCamelCase     ):
				snake_case_      =		"marian"
				snake_case_      =		["past_key_values"]
				snake_case_      =		{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


				def __init__(		self    :					List[Any]       ,				lowercase_    :					Optional[Any]=5_8101       ,				lowercase_    :					Dict=None       ,				lowercase_    :					List[str]=1024       ,				lowercase_    :					Optional[Any]=12       ,				lowercase_    :					int=4096       ,				lowercase_    :					Any=16       ,				lowercase_    :					Optional[int]=12       ,				lowercase_    :					str=4096       ,				lowercase_    :					Union[str, Any]=16       ,				lowercase_    :					Dict=0.0       ,				lowercase_    :					Union[str, Any]=0.0       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Union[str, Any]=True       ,				lowercase_    :					int="gelu"       ,				lowercase_    :					Dict=1024       ,				lowercase_    :					int=0.1       ,				lowercase_    :					Tuple=0.0       ,				lowercase_    :					Tuple=0.0       ,				lowercase_    :					Tuple=0.02       ,				lowercase_    :					int=5_8100       ,				lowercase_    :					Optional[Any]=False       ,				lowercase_    :					Any=5_8100       ,				lowercase_    :					Optional[int]=0       ,				lowercase_    :					Dict=0       ,				lowercase_    :					List[str]=True       ,				**lowercase_    :					Any       ,				):
									snake_case_									=				vocab_size
									snake_case_									=				decoder_vocab_size or vocab_size
									snake_case_									=				max_position_embeddings
									snake_case_									=				d_model
									snake_case_									=				encoder_ffn_dim
									snake_case_									=				encoder_layers
									snake_case_									=				encoder_attention_heads
									snake_case_									=				decoder_ffn_dim
									snake_case_									=				decoder_layers
									snake_case_									=				decoder_attention_heads
									snake_case_									=				dropout
									snake_case_									=				attention_dropout
									snake_case_									=				activation_dropout
									snake_case_									=				activation_function
									snake_case_									=				init_std
									snake_case_									=				encoder_layerdrop
									snake_case_									=				decoder_layerdrop
									snake_case_									=				use_cache
									snake_case_									=				encoder_layers
									snake_case_									=				scale_embedding  # scale factor will be sqrt(d_model) if True
									snake_case_									=				share_encoder_decoder_embeddings
									super().__init__(
									    pad_token_id=lowercase_       ,				eos_token_id=lowercase_       ,				is_encoder_decoder=lowercase_       ,				decoder_start_token_id=lowercase_       ,				forced_eos_token_id=lowercase_       ,				**lowercase_       ,				)







class 						a				(   _lowerCamelCase     ):


				@property
				# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.inputs
				def   A_ (		self    :					Union[str, Any]					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														    ]					)

														if self.use_past:
																			snake_case_									=				{0: '''batch'''}
																			snake_case_									=				{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
														else:
																			snake_case_									=				{0: '''batch''', 1: '''decoder_sequence'''}
																			snake_case_									=				{0: '''batch''', 1: '''decoder_sequence'''}

														if self.use_past:
																			self.fill_with_past_key_values_(lowercase_       ,				direction='''inputs'''					)
									elif self.task == "causal-lm":
														# TODO: figure this case out.
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														    ]					)
														if self.use_past:
																			snake_case_			,snake_case_									=				self.num_layers
																			for i in range(lowercase_					):
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
									else:
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
														        ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
														    ]					)

									return common_inputs


				@property
				# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
				def   A_ (		self    :					Dict					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				super().outputs
									else:
														snake_case_									=				super(lowercase_       ,				self					).outputs
														if self.use_past:
																			snake_case_			,snake_case_									=				self.num_layers
																			for i in range(lowercase_					):
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
									return common_outputs


				def   A_ (		self    :					Dict       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)

									# Generate decoder inputs
									snake_case_									=				seq_length if not self.use_past else 1
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)
									snake_case_									=				{F"decoder_{name}": tensor for name, tensor in decoder_inputs.items()}
									snake_case_									=				dict(**lowercase_       ,				**lowercase_					)

									if self.use_past:
														if not is_torch_available():
																			raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
														else:
																			import torch
														snake_case_			,snake_case_									=				common_inputs['''input_ids'''].shape
														snake_case_									=				common_inputs['''decoder_input_ids'''].shape[1]
														snake_case_			,snake_case_									=				self.num_attention_heads
														snake_case_									=				(
														    batch,
														    num_encoder_attention_heads,
														    encoder_seq_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)
														snake_case_									=				decoder_seq_length + 3
														snake_case_									=				(
														    batch,
														    num_decoder_attention_heads,
														    decoder_past_length,
														    self._config.hidden_size // num_decoder_attention_heads,
														)

														snake_case_									=				torch.cat(
														    [common_inputs['''decoder_attention_mask'''], torch.ones(lowercase_       ,				lowercase_					)]       ,				dim=1					)

														snake_case_									=				[]
														# If the number of encoder and decoder layers are present in the model configuration, both are considered
														snake_case_			,snake_case_									=				self.num_layers
														snake_case_									=				min(lowercase_       ,				lowercase_					)
														snake_case_									=				max(lowercase_       ,				lowercase_					) - min_num_layers
														snake_case_									=				'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

														for _ in range(lowercase_					):
																			common_inputs["past_key_values"].append(
																			    (
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			    )					)
														# TODO: test this.
														snake_case_									=				encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
														for _ in range(lowercase_       ,				lowercase_					):
																			common_inputs["past_key_values"].append((torch.zeros(lowercase_					), torch.zeros(lowercase_					))					)
									return common_inputs


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)

									if self.use_past:
														if not is_torch_available():
																			raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
														else:
																			import torch
														snake_case_			,snake_case_									=				common_inputs['''input_ids'''].shape
														# Not using the same length for past_key_values
														snake_case_									=				seqlen + 2
														snake_case_			,snake_case_									=				self.num_layers
														snake_case_			,snake_case_									=				self.num_attention_heads
														snake_case_									=				(
														    batch,
														    num_encoder_attention_heads,
														    past_key_values_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)

														snake_case_									=				common_inputs['''attention_mask'''].dtype
														snake_case_									=				torch.cat(
														    [common_inputs['''attention_mask'''], torch.ones(lowercase_       ,				lowercase_       ,				dtype=lowercase_					)]       ,				dim=1					)
														snake_case_									=				[
														    (torch.zeros(lowercase_					), torch.zeros(lowercase_					)) for _ in range(lowercase_					)
														]
									return common_inputs


				def   A_ (		self    :					List[str]       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									# Copied from OnnxConfig.generate_dummy_inputs
									# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
									# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
									snake_case_									=				compute_effective_axis_dimension(
									    lowercase_       ,				fixed_dimension=OnnxConfig.default_fixed_batch       ,				num_token_to_add=0					)

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									snake_case_									=				tokenizer.num_special_tokens_to_add(lowercase_					)
									snake_case_									=				compute_effective_axis_dimension(
									    lowercase_       ,				fixed_dimension=OnnxConfig.default_fixed_sequence       ,				num_token_to_add=lowercase_					)

									# Generate dummy inputs according to compute batch and sequence
									snake_case_									=				[''' '''.join([tokenizer.unk_token]					) * seq_length] * batch_size
									snake_case_									=				dict(tokenizer(lowercase_       ,				return_tensors=lowercase_					)					)
									return common_inputs


				def   A_ (		self    :					Any       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				self._generate_dummy_inputs_for_default_and_seqaseq_lm(
														    lowercase_       ,				batch_size=lowercase_       ,				seq_length=lowercase_       ,				is_pair=lowercase_       ,				framework=lowercase_					)

									else:
														snake_case_									=				self._generate_dummy_inputs_for_causal_lm(
														    lowercase_       ,				batch_size=lowercase_       ,				seq_length=lowercase_       ,				is_pair=lowercase_       ,				framework=lowercase_					)

									return common_inputs


				def   A_ (		self    :					Dict       ,				lowercase_    :					List[str]       ,				lowercase_    :					List[str]       ,				lowercase_    :					int       ,				lowercase_    :					List[str]					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				super()._flatten_past_key_values_(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)
									else:
														snake_case_									=				super(lowercase_       ,				self					)._flatten_past_key_values_(
														    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)


				@property
				def   A_ (		self    :					List[str]					):
									return 1e-4

