

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


A__							=				logging.get_logger(__name__)

A__							=				{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

A__							=				{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

A__							=				{
    "Salesforce/codegen-350M-mono": 2048,
}




@lru_cache()
def        _lowercase  (					)					->						Union[str, Any]:




					'''simple docstring'''







					__magic_name__				=     (
					    list(range(ord('!'					)       ,ord('~'					) + 1					)					) + list(range(ord('¡'					)       ,ord('¬'					) + 1					)					) + list(range(ord('®'					)       ,ord('ÿ'					) + 1					)					)
					)
					__magic_name__				=     bs[:]
					__magic_name__				=     0
					for b in range(2**8					):
										if b not in bs:
															bs.append(a_					)
															cs.append(2**8 + n					)
															n += 1
					__magic_name__				=     [chr(a_					) for n in cs]
					return dict(zip(a_       ,a_					)					)




def        _lowercase  (					a_					:       Union[str, Any]					)					->						Optional[int]:




					'''simple docstring'''







					__magic_name__				=     set()
					__magic_name__				=     word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)					)
										__magic_name__				=     char
					return pairs


class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Tuple												=	VOCAB_FILES_NAMES
	_lowercase	: List[str]												=	PRETRAINED_VOCAB_FILES_MAP
	_lowercase	: int												=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowercase	: Tuple												=	["input_ids", "attention_mask"]







	def __init__( self:			Tuple	,			__UpperCamelCase:			Optional[Any]	,			__UpperCamelCase:			Union[str, Any]	,			__UpperCamelCase:			List[str]="replace"	,			__UpperCamelCase:			str="<|endoftext|>"	,			__UpperCamelCase:			Optional[int]="<|endoftext|>"	,			__UpperCamelCase:			int="<|endoftext|>"	,			__UpperCamelCase:			Tuple=None	,			__UpperCamelCase:			Tuple=False	,			__UpperCamelCase:			Optional[int]=False	,			**__UpperCamelCase:			Any	,			):

						'''simple docstring'''





						__magic_name__				=     AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase   ) if isinstance(__UpperCamelCase	,			__UpperCamelCase   ) else bos_token
						__magic_name__				=     AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase   ) if isinstance(__UpperCamelCase	,			__UpperCamelCase   ) else eos_token
						__magic_name__				=     AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase   ) if isinstance(__UpperCamelCase	,			__UpperCamelCase   ) else unk_token
						__magic_name__				=     AddedToken(__UpperCamelCase	,			lstrip=__UpperCamelCase	,			rstrip=__UpperCamelCase   ) if isinstance(__UpperCamelCase	,			__UpperCamelCase   ) else pad_token
						super().__init__(
						    errors=__UpperCamelCase	,			unk_token=__UpperCamelCase	,			bos_token=__UpperCamelCase	,			eos_token=__UpperCamelCase	,			pad_token=__UpperCamelCase	,			add_prefix_space=__UpperCamelCase	,			add_bos_token=__UpperCamelCase	,			**__UpperCamelCase	,			)
						__magic_name__				=     add_bos_token

						with open(__UpperCamelCase	,			encoding='utf-8'   ) as vocab_handle:
											__magic_name__				=     json.load(__UpperCamelCase   )
						__magic_name__				=     {v: k for k, v in self.encoder.items()}
						__magic_name__				=     errors  # how to handle errors in decoding
						__magic_name__				=     bytes_to_unicode()
						__magic_name__				=     {v: k for k, v in self.byte_encoder.items()}
						with open(__UpperCamelCase	,			encoding='utf-8'   ) as merges_handle:
											__magic_name__				=     merges_handle.read().split('\n'   )[1:-1]
						__magic_name__				=     [tuple(merge.split()   ) for merge in bpe_merges]
						__magic_name__				=     dict(zip(__UpperCamelCase	,			range(len(__UpperCamelCase   )   )   )   )
						__magic_name__				=     {}
						__magic_name__				=     add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						__magic_name__				=     re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]   ):

						'''simple docstring'''





						return len(self.encoder   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						return dict(self.encoder	,			**self.added_tokens_encoder   )







	def 						_SCREAMING_SNAKE_CASE				( self:			str	,			__UpperCamelCase:			Union[str, Any]   ):

						'''simple docstring'''





						if token in self.cache:
											return self.cache[token]
						__magic_name__				=     tuple(__UpperCamelCase   )
						__magic_name__				=     get_pairs(__UpperCamelCase   )

						if not pairs:
											return token

						while True:
											__magic_name__				=     min(__UpperCamelCase	,			key=lambda __UpperCamelCase   : self.bpe_ranks.get(__UpperCamelCase	,			float('inf'   )   )   )
											if bigram not in self.bpe_ranks:
																break
											__magic_name__,						__magic_name__				=     bigram
											__magic_name__				=     []
											__magic_name__				=     0
											while i < len(__UpperCamelCase   ):
																try:
																					__magic_name__				=     word.index(__UpperCamelCase	,			__UpperCamelCase   )
																except ValueError:
																					new_word.extend(word[i:]   )
																					break
																else:
																					new_word.extend(word[i:j]   )
																					__magic_name__				=     j

																if word[i] == first and i < len(__UpperCamelCase   ) - 1 and word[i + 1] == second:
																					new_word.append(first + second   )
																					i += 2
																else:
																					new_word.append(word[i]   )
																					i += 1
											__magic_name__				=     tuple(__UpperCamelCase   )
											__magic_name__				=     new_word
											if len(__UpperCamelCase   ) == 1:
																break
											else:
																__magic_name__				=     get_pairs(__UpperCamelCase   )
						__magic_name__				=     ' '.join(__UpperCamelCase   )
						__magic_name__				=     word
						return word







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[Any]	,			__UpperCamelCase:			Dict	,			__UpperCamelCase:			str=None   ):

						'''simple docstring'''





						if self.add_bos_token:
											__magic_name__				=     [self.bos_token_id]
						else:
											__magic_name__				=     []

						__magic_name__				=     bos_token_ids + token_ids_a

						if token_ids_a is None:
											return output

						return output + bos_token_ids + token_ids_a







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[Any]	,			__UpperCamelCase:			List[Any]   ):

						'''simple docstring'''





						__magic_name__				=     []
						for token in re.findall(self.pat	,			__UpperCamelCase   ):
											__magic_name__				=     ''.join(
											    self.byte_encoder[b] for b in token.encode('utf-8'   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase   ).split(' '   )   )
						return bpe_tokens







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]	,			__UpperCamelCase:			Optional[Any]   ):

						'''simple docstring'''





						return self.encoder.get(__UpperCamelCase	,			self.encoder.get(self.unk_token   )   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			Tuple   ):

						'''simple docstring'''





						return self.decoder.get(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]	,			__UpperCamelCase:			Optional[Any]   ):

						'''simple docstring'''





						__magic_name__				=     ''.join(__UpperCamelCase   )
						__magic_name__				=     bytearray([self.byte_decoder[c] for c in text]   ).decode('utf-8'	,			errors=self.errors   )
						return text







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			str	,			__UpperCamelCase:			Optional[str] = None   ):

						'''simple docstring'''





						if not os.path.isdir(__UpperCamelCase   ):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
											return
						__magic_name__				=     os.path.join(
						    __UpperCamelCase	,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
						__magic_name__				=     os.path.join(
						    __UpperCamelCase	,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

						with open(__UpperCamelCase	,			'w'	,			encoding='utf-8'   ) as f:
											f.write(json.dumps(self.encoder	,			indent=2	,			sort_keys=__UpperCamelCase	,			ensure_ascii=__UpperCamelCase   ) + '\n'   )

						__magic_name__				=     0
						with open(__UpperCamelCase	,			'w'	,			encoding='utf-8'   ) as writer:
											writer.write('#version: 0.2\n'   )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,			key=lambda __UpperCamelCase   : kv[1]   ):
																if index != token_index:
																					logger.warning(
																					    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																					    ' Please check that the tokenizer is not corrupted!'   )
																					__magic_name__				=     token_index
																writer.write(' '.join(__UpperCamelCase   ) + '\n'   )
																index += 1

						return vocab_file, merge_file







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			Union[str, Any]	,			__UpperCamelCase:			int=False	,			**__UpperCamelCase:			Optional[Any]   ):

						'''simple docstring'''





						__magic_name__				=     kwargs.pop('add_prefix_space'	,			self.add_prefix_space   )
						if is_split_into_words or add_prefix_space:
											__magic_name__				=     ' ' + text
						return (text, kwargs)







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple	,			__UpperCamelCase:			Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]	,			__UpperCamelCase:			bool = False	,			__UpperCamelCase:			bool = None	,			__UpperCamelCase:			Optional[List[str]] = None	,			**__UpperCamelCase:			Optional[int]	,			):

						'''simple docstring'''





						__magic_name__				=     super()._decode(
						    token_ids=__UpperCamelCase	,			skip_special_tokens=__UpperCamelCase	,			clean_up_tokenization_spaces=__UpperCamelCase	,			**__UpperCamelCase	,			)

						if truncate_before_pattern is not None and len(__UpperCamelCase   ) > 0:
											__magic_name__				=     self.truncate(__UpperCamelCase	,			__UpperCamelCase   )

						return decoded_text







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[Any]	,			__UpperCamelCase:			int	,			__UpperCamelCase:			Union[str, Any]   ):

						'''simple docstring'''





						def find_re(__UpperCamelCase:			Union[str, Any]	,			__UpperCamelCase:			Optional[int]	,			__UpperCamelCase:			List[str]   ):
											__magic_name__				=     pattern.search(__UpperCamelCase	,			__UpperCamelCase   )
											return m.start() if m else -1

						__magic_name__				=     [re.compile(__UpperCamelCase	,			re.MULTILINE   ) for pattern in truncate_before_pattern]

						__magic_name__				=     list(re.finditer('^print'	,			__UpperCamelCase	,			re.MULTILINE   )   )

						if len(__UpperCamelCase   ) > 1:
											__magic_name__				=     completion[: prints[1].start()]

						__magic_name__				=     list(re.finditer('^def'	,			__UpperCamelCase	,			re.MULTILINE   )   )

						if len(__UpperCamelCase   ) > 1:
											__magic_name__				=     completion[: defs[1].start()]

						__magic_name__				=     0

						__magic_name__				=     [
						    pos for pos in [find_re(__UpperCamelCase	,			__UpperCamelCase	,			__UpperCamelCase   ) for terminal in terminals] if pos != -1
						]

						if len(__UpperCamelCase   ) > 0:
											return completion[: min(__UpperCamelCase   )]
						else:
											return completion







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__							=				logging.get_logger(__name__)

A__							=				{
    "nghuyong/ernie-1.0-base-zh": "https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json",
    "nghuyong/ernie-2.0-base-en": "https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json",
    "nghuyong/ernie-2.0-large-en": "https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json",
    "nghuyong/ernie-3.0-base-zh": "https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-medium-zh": "https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-mini-zh": "https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-micro-zh": "https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-nano-zh": "https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json",
    "nghuyong/ernie-gram-zh": "https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json",
    "nghuyong/ernie-health-zh": "https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json",
}

class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Dict												=	"ernie"







	def __init__( self:			List[Any]	,			__UpperCamelCase:			Dict=3_05_22	,			__UpperCamelCase:			str=7_68	,			__UpperCamelCase:			Tuple=12	,			__UpperCamelCase:			int=12	,			__UpperCamelCase:			Optional[Any]=30_72	,			__UpperCamelCase:			str="gelu"	,			__UpperCamelCase:			Any=0.1	,			__UpperCamelCase:			int=0.1	,			__UpperCamelCase:			int=5_12	,			__UpperCamelCase:			Any=2	,			__UpperCamelCase:			Tuple=3	,			__UpperCamelCase:			Any=False	,			__UpperCamelCase:			Optional[int]=0.02	,			__UpperCamelCase:			List[Any]=1E-12	,			__UpperCamelCase:			Union[str, Any]=0	,			__UpperCamelCase:			Tuple="absolute"	,			__UpperCamelCase:			Union[str, Any]=True	,			__UpperCamelCase:			Optional[Any]=None	,			**__UpperCamelCase:			List[Any]	,			):

						'''simple docstring'''





						super().__init__(pad_token_id=__UpperCamelCase	,			**__UpperCamelCase   )

						__magic_name__				=     vocab_size
						__magic_name__				=     hidden_size
						__magic_name__				=     num_hidden_layers
						__magic_name__				=     num_attention_heads
						__magic_name__				=     hidden_act
						__magic_name__				=     intermediate_size
						__magic_name__				=     hidden_dropout_prob
						__magic_name__				=     attention_probs_dropout_prob
						__magic_name__				=     max_position_embeddings
						__magic_name__				=     type_vocab_size
						__magic_name__				=     task_type_vocab_size
						__magic_name__				=     use_task_id
						__magic_name__				=     initializer_range
						__magic_name__				=     layer_norm_eps
						__magic_name__				=     position_embedding_type
						__magic_name__				=     use_cache
						__magic_name__				=     classifier_dropout




class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						if self.task == "multiple-choice":
											__magic_name__				=     {0: 'batch', 1: 'choice', 2: 'sequence'}
						else:
											__magic_name__				=     {0: 'batch', 1: 'sequence'}
						return OrderedDict(
						    [
						        ('input_ids', dynamic_axis),
						        ('attention_mask', dynamic_axis),
						        ('token_type_ids', dynamic_axis),
						        ('task_type_ids', dynamic_axis),
						    ]   )




