





'''simple docstring'''







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_					:    List[Any]      	=			logging.get_logger(__name__)

lowerCAmelCase_					:    str      	=			'▁'

lowerCAmelCase_					:    Dict      	=			{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

lowerCAmelCase_					:    str      	=			{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

lowerCAmelCase_					:    Any      	=			{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

lowerCAmelCase_					:    Any      	=			{
    'ernie-m-base': 5_14,
    'ernie-m-large': 5_14,
}

lowerCAmelCase_					:    Any      	=			{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            =["input_ids"]

				__a            =VOCAB_FILES_NAMES
				__a            =PRETRAINED_INIT_CONFIGURATION
				__a            =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a            =PRETRAINED_VOCAB_FILES_MAP
				__a            =RESOURCE_FILES_NAMES




				def __init__(  self     :					Tuple					,				__a     :					Optional[Any]					,				__a     :					Union[str, Any]=None					,				__a     :					int=False					,				__a     :					Union[str, Any]="utf8"					,				__a     :					Union[str, Any]="[UNK]"					,				__a     :					Any="[SEP]"					,				__a     :					Tuple="[PAD]"					,				__a     :					Union[str, Any]="[CLS]"					,				__a     :					List[Any]="[MASK]"					,				__a     :					Optional[Dict[str, Any]] = None					,				**__a     :					Tuple					,				):
								# Mask token behave like a normal word, i.e. include the space before it and
								# is included in the raw text, there should be a match in a non-normalized sentence.

								_a								= {} if sp_model_kwargs is None else sp_model_kwargs
								super().__init__(
								    do_lower_case=__a					,				unk_token=__a					,				sep_token=__a					,				pad_token=__a					,				cls_token=__a					,				mask_token=__a					,				vocab_file=__a					,				encoding=__a					,				sp_model_kwargs=self.sp_model_kwargs					,				**__a					,				)
								_a								= do_lower_case
								_a								= sentencepiece_model_ckpt
								_a								= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(__a					)

								# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
								if vocab_file is not None:
												_a								= self.load_vocab(filepath=__a					)
								else:
												_a								= {self.sp_model.id_to_piece(__a					): id for id in range(self.sp_model.get_piece_size()					)}
								_a								= {v: k for k, v in self.vocab.items()}




				def   UpperCamelCase__   (  self     :					List[Any]					,				__a     :					Any					):
								if text is None:
												return None

								_a								= self.tokenize(__a					)
								_a	,					_a								= "", []

								for i, ch in enumerate(__a					):
												if ch in self.SP_CHAR_MAPPING:
																_a								= self.SP_CHAR_MAPPING.get(__a					)
												else:
																_a								= unicodedata.normalize("NFKC"					,				__a					)
												if self.is_whitespace(__a					):
																continue
												normalized_text += ch
												char_mapping.extend([i] * len(__a					)					)

								_a	,					_a	,					_a								= normalized_text, [], 0

								if self.do_lower_case:
												_a								= text.lower()

								for token in split_tokens:
												if token[:1] == "▁":
																_a								= token[1:]
												_a								= text[offset:].index(__a					) + offset
												_a								= start + len(__a					)

												token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
												_a								= end
								return token_mapping




				@property
				def   UpperCamelCase__   (  self     :					int					):
								return len(self.vocab					)




				def   UpperCamelCase__   (  self     :					Dict					):
								return dict(self.vocab					,				**self.added_tokens_encoder					)




				def __getstate__(  self     :					Optional[int]					):
								_a								= self.__dict__.copy()
								_a								= None
								return state




				def __setstate__(  self     :					List[Any]					,				__a     :					Any					):
								_a								= d

								# for backward compatibility
								if not hasattr(self					,				"sp_model_kwargs"					):
												_a								= {}

								_a								= spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(self.sentencepiece_model_ckpt					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					Optional[int]					):
								return "".join((self.SP_CHAR_MAPPING.get(__a					,				__a					) for c in text)					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					List[Any]					,				__a     :					List[str]=False					,				__a     :					Tuple=64					,				__a     :					Union[str, Any]=0.1					):

								if self.sp_model_kwargs.get("enable_sampling"					) is True:
												_a								= True
								if self.sp_model_kwargs.get("alpha"					) is not None:
												_a								= self.sp_model_kwargs.get("alpha"					)
								if self.sp_model_kwargs.get("nbest_size"					) is not None:
												_a								= self.sp_model_kwargs.get("nbest_size"					)

								if not enable_sampling:
												_a								= self.sp_model.EncodeAsPieces(__a					)
								else:
												_a								= self.sp_model.SampleEncodeAsPieces(__a					,				__a					,				__a					)
								_a								= []
								for pi, piece in enumerate(__a					):
												if piece == SPIECE_UNDERLINE:
																if not pieces[pi + 1].startswith(__a					) and pi != 0:
																				new_pieces.append(__a					)
																				continue
																else:
																				continue
												_a								= 0
												for i, chunk in enumerate(__a					):
																if chunk == SPIECE_UNDERLINE:
																				continue
																if self.is_ch_char(__a					) or self.is_punct(__a					):
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]					)
																				new_pieces.append(__a					)
																				_a								= i + 1
																elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]					)
																				_a								= i
																elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]					)
																				_a								= i
												if len(__a					) > lst_i:
																new_pieces.append(piece[lst_i:]					)
								return new_pieces




				def   UpperCamelCase__   (  self     :					Tuple					,				__a     :					Optional[int]					):
								_a								= "".join(__a					).replace(__a					,				" "					).strip()
								return out_string




				def   UpperCamelCase__   (  self     :					Any					,				__a     :					Dict					):
								_a								= self.convert_ids_to_tokens(__a					)
								_a								= "".join(__a					).replace(__a					,				" "					).strip()
								return out_string




				def   UpperCamelCase__   (  self     :					str					,				__a     :					Optional[Any]					):
								return self.vocab.get(__a					,				self.vocab.get(self.unk_token					)					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					Any					):
								return self.reverse_vocab.get(__a					,				self.unk_token					)




				def   UpperCamelCase__   (  self     :					int					,				__a     :					int					,				__a     :					Optional[int]=None					):
								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								_a								= [self.cls_token_id]
								_a								= [self.sep_token_id]
								return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




				def   UpperCamelCase__   (  self     :					Optional[int]					,				__a     :					Any					,				__a     :					Optional[int]=None					):
								if offset_mapping_a is None:
												return [(0, 0)] + offset_mapping_a + [(0, 0)]

								return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				__a     :					Optional[Any]					,				__a     :					Any=None					,				__a     :					Optional[Any]=False					):

								if already_has_special_tokens:
												if token_ids_a is not None:
																raise ValueError(
																    "You should not supply a second sequence if the provided sequence of "
																    "ids is already formatted with special tokens for the model."					)
												return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
												return [1] + ([0] * len(__a					)) + [1, 1] + ([0] * len(__a					)) + [1]
								return [1] + ([0] * len(__a					)) + [1]




				def   UpperCamelCase__   (  self     :					List[Any]					,				__a     :					List[int]					,				__a     :					Optional[List[int]] = None					):
								# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
								if token_ids_a is None:
												# [CLS] X [SEP]
												return (len(__a					) + 2) * [0]

								# [CLS] A [SEP] [SEP] B [SEP]
								return [0] * (len(__a					) + 1) + [1] * (len(__a					) + 3)




				def   UpperCamelCase__   (  self     :					Any					,				__a     :					Tuple					):
								if "\u4e00" <= char <= "\u9fff":
												return True
								return False




				def   UpperCamelCase__   (  self     :					int					,				__a     :					int					):
								if ("a" <= char <= "z") or ("A" <= char <= "Z"):
												return True
								return False




				def   UpperCamelCase__   (  self     :					List[str]					,				__a     :					List[Any]					):
								if char in ",;:.?!~，；：。？！《》【】":
												return True
								return False




				def   UpperCamelCase__   (  self     :					Any					,				__a     :					Optional[Any]					):
								if char == " " or char == "\t" or char == "\n" or char == "\r":
												return True
								if len(__a					) == 1:
												_a								= unicodedata.category(__a					)
												if cat == "Zs":
																return True
								return False




				def   UpperCamelCase__   (  self     :					List[Any]					,				__a     :					int					):
								_a								= {}
								with io.open(__a					,				"r"					,				encoding="utf-8"					) as f:
												for index, line in enumerate(__a					):
																_a								= line.rstrip("\n"					)
																_a								= int(__a					)

								return token_to_idx




				def   UpperCamelCase__   (  self     :					Tuple					,				__a     :					str					,				__a     :					Optional[str] = None					):
								_a								= 0
								if os.path.isdir(__a					):
												_a								= os.path.join(
												    __a					,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
								else:
												_a								= (filename_prefix + "-" if filename_prefix else "") + save_directory
								with open(__a					,				"w"					,				encoding="utf-8"					) as writer:
												for token, token_index in sorted(self.vocab.items()					,				key=lambda __a					: kv[1]					):
																if index != token_index:
																				logger.warning(
																				    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																				    " Please check that the vocabulary is not corrupted!"					)
																				_a								= token_index
																writer.write(token + "\n"					)
																index += 1

								_a								= os.path.join(__a					,				"sentencepiece.bpe.model"					)
								with open(__a					,				"wb"					) as fi:
												_a								= self.sp_model.serialized_model_proto()
												fi.write(__a					)

								return (vocab_file,)







'''simple docstring'''







import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available
class 				__SCREAMING_SNAKE_CASE  (lowerCamelCase_      ):






				"""simple docstring"""







				__a            =['image_processor', 'tokenizer']
				__a            ='OwlViTImageProcessor'
				__a            =('CLIPTokenizer', 'CLIPTokenizerFast')




				def __init__(  self     :					List[Any]					,				__a     :					str=None					,				__a     :					List[str]=None					,				**__a     :					List[Any]					):
								_a								= None
								if "feature_extractor" in kwargs:
												warnings.warn(
												    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
												    " instead."					,				__a					,				)
												_a								= kwargs.pop("feature_extractor"					)

								_a								= image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
												raise ValueError("You need to specify an `image_processor`."					)
								if tokenizer is None:
												raise ValueError("You need to specify a `tokenizer`."					)

								super().__init__(__a					,				__a					)




				def __call__(  self     :					Union[str, Any]					,				__a     :					Any=None					,				__a     :					List[str]=None					,				__a     :					int=None					,				__a     :					Optional[int]="max_length"					,				__a     :					List[str]="np"					,				**__a     :					Any					):

								if text is None and query_images is None and images is None:
												raise ValueError(
												    "You have to specify at least one text or query image or image. All three cannot be none."					)

								if text is not None:
												if isinstance(__a					,				__a					) or (isinstance(__a					,				__a					) and not isinstance(text[0]					,				__a					)):
																_a								= [self.tokenizer(__a					,				padding=__a					,				return_tensors=__a					,				**__a					)]

												elif isinstance(__a					,				__a					) and isinstance(text[0]					,				__a					):
																_a								= []

																# Maximum number of queries across batch
																_a								= max([len(__a					) for t in text]					)

																# Pad all batch samples to max number of text queries
																for t in text:
																				if len(__a					) != max_num_queries:
																								_a								= t + [" "] * (max_num_queries - len(__a					))

																				_a								= self.tokenizer(__a					,				padding=__a					,				return_tensors=__a					,				**__a					)
																				encodings.append(__a					)
												else:
																raise TypeError("Input text should be a string, a list of strings or a nested list of strings"					)

												if return_tensors == "np":
																_a								= np.concatenate([encoding["input_ids"] for encoding in encodings]					,				axis=0					)
																_a								= np.concatenate([encoding["attention_mask"] for encoding in encodings]					,				axis=0					)

												elif return_tensors == "jax" and is_flax_available():
																import jax.numpy as jnp

																_a								= jnp.concatenate([encoding["input_ids"] for encoding in encodings]					,				axis=0					)
																_a								= jnp.concatenate([encoding["attention_mask"] for encoding in encodings]					,				axis=0					)

												elif return_tensors == "pt" and is_torch_available():
																import torch

																_a								= torch.cat([encoding["input_ids"] for encoding in encodings]					,				dim=0					)
																_a								= torch.cat([encoding["attention_mask"] for encoding in encodings]					,				dim=0					)

												elif return_tensors == "tf" and is_tf_available():
																import tensorflow as tf

																_a								= tf.stack([encoding["input_ids"] for encoding in encodings]					,				axis=0					)
																_a								= tf.stack([encoding["attention_mask"] for encoding in encodings]					,				axis=0					)

												else:
																raise ValueError("Target return tensor type could not be returned"					)

												_a								= BatchEncoding()
												_a								= input_ids
												_a								= attention_mask

								if query_images is not None:
												_a								= BatchEncoding()
												_a								= self.image_processor(
												    __a					,				return_tensors=__a					,				**__a					).pixel_values
												_a								= query_pixel_values

								if images is not None:
												_a								= self.image_processor(__a					,				return_tensors=__a					,				**__a					)

								if text is not None and images is not None:
												_a								= image_features.pixel_values
												return encoding
								elif query_images is not None and images is not None:
												_a								= image_features.pixel_values
												return encoding
								elif text is not None or query_images is not None:
												return encoding
								else:
												return BatchEncoding(data=dict(**__a					)					,				tensor_type=__a					)




				def   UpperCamelCase__   (  self     :					List[str]					,				*__a     :					Union[str, Any]					,				**__a     :					int					):
								return self.image_processor.post_process(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					Optional[int]					,				*__a     :					Optional[Any]					,				**__a     :					List[str]					):
								return self.image_processor.post_process_object_detection(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					,				*__a     :					Dict					,				**__a     :					Union[str, Any]					):
								return self.image_processor.post_process_image_guided_detection(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					str					,				*__a     :					Tuple					,				**__a     :					Tuple					):
								return self.tokenizer.batch_decode(*__a					,				**__a					)




				def   UpperCamelCase__   (  self     :					List[str]					,				*__a     :					List[Any]					,				**__a     :					Optional[int]					):
								return self.tokenizer.decode(*__a					,				**__a					)




				@property
				def   UpperCamelCase__   (  self     :					List[str]					):
								warnings.warn(
								    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."					,				__a					,				)
								return self.image_processor_class




				@property
				def   UpperCamelCase__   (  self     :					str					):
								warnings.warn(
								    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."					,				__a					,				)
								return self.image_processor
