



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase: List[Any]      			=		logging.get_logger(__name__)

_lowercase: str      			=		'▁'

_lowercase: Dict      			=		{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

_lowercase: str      			=		{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

_lowercase: Any      			=		{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

_lowercase: Any      			=		{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

_lowercase: Any      			=		{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}



class 	_lowercase							(   lowerCamelCase_	):





	"""simple docstring"""
	__A							=							["input_ids"]

	__A							=							VOCAB_FILES_NAMES
	__A							=							PRETRAINED_INIT_CONFIGURATION
	__A							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A							=							PRETRAINED_VOCAB_FILES_MAP
	__A							=							RESOURCE_FILES_NAMES




	def __init__(self		,       lowerCamelCase_		,       lowerCamelCase_=None		,       lowerCamelCase_=False		,       lowerCamelCase_="utf8"		,       lowerCamelCase_="[UNK]"		,       lowerCamelCase_="[SEP]"		,       lowerCamelCase_="[PAD]"		,       lowerCamelCase_="[CLS]"		,       lowerCamelCase_="[MASK]"		,       lowerCamelCase_ = None		,       **lowerCamelCase_		,       ):
						"""simple docstring"""

						a					     =   {} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=__a		,       unk_token=__a		,       sep_token=__a		,       pad_token=__a		,       cls_token=__a		,       mask_token=__a		,       vocab_file=__a		,       encoding=__a		,       sp_model_kwargs=self.sp_model_kwargs		,       **__a		,       )
						a					     =   do_lower_case
						a					     =   sentencepiece_model_ckpt
						a					     =   spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(__a		)

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
											a					     =   self.load_vocab(filepath=__a		)
						else:
											a					     =   {self.sp_model.id_to_piece(__a		): id for id in range(self.sp_model.get_piece_size()		)}
						a					     =   {v: k for k, v in self.vocab.items()}




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						if text is None:
											return None

						a					     =   self.tokenize(__a		)
						a     ,				a					     =   "", []

						for i, ch in enumerate(__a		):
											if ch in self.SP_CHAR_MAPPING:
																a					     =   self.SP_CHAR_MAPPING.get(__a		)
											else:
																a					     =   unicodedata.normalize("NFKC"		,       __a		)
											if self.is_whitespace(__a		):
																continue
											normalized_text += ch
											char_mapping.extend([i] * len(__a		)		)

						a     ,				a     ,				a					     =   normalized_text, [], 0

						if self.do_lower_case:
											a					     =   text.lower()

						for token in split_tokens:
											if token[:1] == "▁":
																a					     =   token[1:]
											a					     =   text[offset:].index(__a		) + offset
											a					     =   start + len(__a		)

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
											a					     =   end
						return token_mapping




	@property
	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						return len(self.vocab		)




	def      UpperCamelCase_       (self		):
						"""simple docstring"""

						return dict(self.vocab		,       **self.added_tokens_encoder		)




	def __getstate__(self		):
						"""simple docstring"""

						a					     =   self.__dict__.copy()
						a					     =   None
						return state




	def __setstate__(self		,       lowerCamelCase_		):
						"""simple docstring"""

						a					     =   d

						# for backward compatibility
						if not hasattr(self		,       "sp_model_kwargs"		):
											a					     =   {}

						a					     =   spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(self.sentencepiece_model_ckpt		)




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						return "".join((self.SP_CHAR_MAPPING.get(__a		,       __a		) for c in text)		)




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_=False		,       lowerCamelCase_=64		,       lowerCamelCase_=0.1		):
						"""simple docstring"""

						if self.sp_model_kwargs.get("enable_sampling"		) is True:
											a					     =   True
						if self.sp_model_kwargs.get("alpha"		) is not None:
											a					     =   self.sp_model_kwargs.get("alpha"		)
						if self.sp_model_kwargs.get("nbest_size"		) is not None:
											a					     =   self.sp_model_kwargs.get("nbest_size"		)

						if not enable_sampling:
											a					     =   self.sp_model.EncodeAsPieces(__a		)
						else:
											a					     =   self.sp_model.SampleEncodeAsPieces(__a		,       __a		,       __a		)
						a					     =   []
						for pi, piece in enumerate(__a		):
											if piece == SPIECE_UNDERLINE:
																if not pieces[pi + 1].startswith(__a		) and pi != 0:
																					new_pieces.append(__a		)
																					continue
																else:
																					continue
											a					     =   0
											for i, chunk in enumerate(__a		):
																if chunk == SPIECE_UNDERLINE:
																					continue
																if self.is_ch_char(__a		) or self.is_punct(__a		):
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																					new_pieces.append(__a		)
																					a					     =   i + 1
																elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																					a					     =   i
																elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																					a					     =   i
											if len(__a		) > lst_i:
																new_pieces.append(piece[lst_i:]		)
						return new_pieces




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						a					     =   "".join(__a		).replace(__a		,       " "		).strip()
						return out_string




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						a					     =   self.convert_ids_to_tokens(__a		)
						a					     =   "".join(__a		).replace(__a		,       " "		).strip()
						return out_string




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						return self.vocab.get(__a		,       self.vocab.get(self.unk_token		)		)




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						return self.reverse_vocab.get(__a		,       self.unk_token		)




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_=None		):
						"""simple docstring"""

						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						a					     =   [self.cls_token_id]
						a					     =   [self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_=None		):
						"""simple docstring"""

						if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_=None		,       lowerCamelCase_=False		):
						"""simple docstring"""

						if already_has_special_tokens:
											if token_ids_a is not None:
																raise ValueError(
																    "You should not supply a second sequence if the provided sequence of "
																    "ids is already formatted with special tokens for the model."		)
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
											return [1] + ([0] * len(__a		)) + [1, 1] + ([0] * len(__a		)) + [1]
						return [1] + ([0] * len(__a		)) + [1]




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_ = None		):
						"""simple docstring"""

						if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(__a		) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(__a		) + 1) + [1] * (len(__a		) + 3)




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						if "\u4e00" <= char <= "\u9fff":
											return True
						return False




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
						return False




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						if char in ",;:.?!~，；：。？！《》【】":
											return True
						return False




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
						if len(__a		) == 1:
											a					     =   unicodedata.category(__a		)
											if cat == "Zs":
																return True
						return False




	def      UpperCamelCase_       (self		,       lowerCamelCase_		):
						"""simple docstring"""

						a					     =   {}
						with io.open(__a		,       "r"		,       encoding="utf-8"		) as f:
											for index, line in enumerate(__a		):
																a					     =   line.rstrip("\n"		)
																a					     =   int(__a		)

						return token_to_idx




	def      UpperCamelCase_       (self		,       lowerCamelCase_		,       lowerCamelCase_ = None		):
						"""simple docstring"""

						a					     =   0
						if os.path.isdir(__a		):
											a					     =   os.path.join(
											    __a		,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
						else:
											a					     =   (filename_prefix + "-" if filename_prefix else "") + save_directory
						with open(__a		,       "w"		,       encoding="utf-8"		) as writer:
											for token, token_index in sorted(self.vocab.items()		,       key=lambda lowerCamelCase_		: kv[1]		):
																if index != token_index:
																					logger.warning(
																					    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																					    " Please check that the vocabulary is not corrupted!"		)
																					a					     =   token_index
																writer.write(token + "\n"		)
																index += 1

						a					     =   os.path.join(__a		,       "sentencepiece.bpe.model"		)
						with open(__a		,       "wb"		) as fi:
											a					     =   self.sp_model.serialized_model_proto()
											fi.write(__a		)

						return (vocab_file,)







'''simple docstring'''







import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCAmelCase_					:    Dict      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    Optional[int]      	=			{
    'ut/deta': 'https://huggingface.co/ut/deta/resolve/main/config.json',
}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            ='deta'
				__a            ={
				    'hidden_size': 'd_model',
				    'num_attention_heads': 'encoder_attention_heads',
				}




				def __init__(  self     :					List[str]					,				__a     :					List[str]=None					,				__a     :					Dict=9_00					,				__a     :					str=20_48					,				__a     :					Tuple=6					,				__a     :					List[str]=20_48					,				__a     :					str=8					,				__a     :					Union[str, Any]=6					,				__a     :					int=10_24					,				__a     :					List[Any]=8					,				__a     :					Dict=0.0					,				__a     :					Tuple=True					,				__a     :					Optional[Any]="relu"					,				__a     :					Tuple=2_56					,				__a     :					Optional[Any]=0.1					,				__a     :					int=0.0					,				__a     :					List[Any]=0.0					,				__a     :					Optional[int]=0.02					,				__a     :					str=1.0					,				__a     :					Dict=True					,				__a     :					Dict=False					,				__a     :					Optional[int]="sine"					,				__a     :					Any=5					,				__a     :					List[str]=4					,				__a     :					Optional[int]=4					,				__a     :					List[str]=True					,				__a     :					str=3_00					,				__a     :					int=True					,				__a     :					int=True					,				__a     :					Tuple=1					,				__a     :					Optional[int]=5					,				__a     :					Tuple=2					,				__a     :					Dict=1					,				__a     :					Optional[int]=1					,				__a     :					Any=5					,				__a     :					Optional[int]=2					,				__a     :					Dict=0.1					,				__a     :					str=0.25					,				**__a     :					Tuple					,				):
								if backbone_config is None:
												logger.info("`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."					)
												_a								= CONFIG_MAPPING["resnet"](out_features=["stage2", "stage3", "stage4"]					)
								else:
												if isinstance(__a					,				__a					):
																_a								= backbone_config.pop("model_type"					)
																_a								= CONFIG_MAPPING[backbone_model_type]
																_a								= config_class.from_dict(__a					)

								_a								= backbone_config
								_a								= num_queries
								_a								= max_position_embeddings
								_a								= d_model
								_a								= encoder_ffn_dim
								_a								= encoder_layers
								_a								= encoder_attention_heads
								_a								= decoder_ffn_dim
								_a								= decoder_layers
								_a								= decoder_attention_heads
								_a								= dropout
								_a								= attention_dropout
								_a								= activation_dropout
								_a								= activation_function
								_a								= init_std
								_a								= init_xavier_std
								_a								= encoder_layerdrop
								_a								= auxiliary_loss
								_a								= position_embedding_type
								# deformable attributes
								_a								= num_feature_levels
								_a								= encoder_n_points
								_a								= decoder_n_points
								_a								= two_stage
								_a								= two_stage_num_proposals
								_a								= with_box_refine
								_a								= assign_first_stage
								if two_stage is True and with_box_refine is False:
												raise ValueError("If two_stage is True, with_box_refine must be True."					)
								# Hungarian matcher
								_a								= class_cost
								_a								= bbox_cost
								_a								= giou_cost
								# Loss coefficients
								_a								= mask_loss_coefficient
								_a								= dice_loss_coefficient
								_a								= bbox_loss_coefficient
								_a								= giou_loss_coefficient
								_a								= eos_coefficient
								_a								= focal_alpha
								super().__init__(is_encoder_decoder=__a					,				**__a					)




				@property
				def   UpperCamelCase__   (  self     :					Optional[Any]					):
								return self.encoder_attention_heads




				@property
				def   UpperCamelCase__   (  self     :					Dict					):
								return self.d_model




				def   UpperCamelCase__   (  self     :					List[str]					):
								_a								= copy.deepcopy(self.__dict__					)
								_a								= self.backbone_config.to_dict()
								_a								= self.__class__.model_type
								return output
