
'''simple docstring'''
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__ :		Tuple					=logging.get_logger(__name__)







class 			__A					(			_SCREAMING_SNAKE_CASE      ):
 lowerCamelCase												=['''input_ids''', '''attention_mask''']
 def __init__( self      :     List[str]					, lowerCamelCase      :     Optional[int]="[PAD]"					, lowerCamelCase      :     Dict="[BOS]"					, lowerCamelCase      :     Dict="[EOS]"					, lowerCamelCase      :     Optional[Any]="[MASK]"					, lowerCamelCase      :     Union[str, Any]="[CLS]"					, lowerCamelCase      :     Any="[SEP]"					, lowerCamelCase      :     int=20_48					, **lowerCamelCase      :     List[str]					, ):



   """simple docstring"""





   __A		:  Dict       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else pad_token
   __A		:  int       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else bos_token
   __A		:  Optional[Any]       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else eos_token
   __A		:  List[Any]       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else mask_token
   __A		:  Dict       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else cls_token
   __A		:  Dict       				=		AddedToken(lowerCamelCase					, lstrip=lowerCamelCase					, rstrip=lowerCamelCase							) if isinstance(lowerCamelCase					, lowerCamelCase							) else sep_token

   super().__init__(
       pad_token=lowerCamelCase					, bos_token=lowerCamelCase					, eos_token=lowerCamelCase					, mask_token=lowerCamelCase					, cls_token=lowerCamelCase					, sep_token=lowerCamelCase					, model_max_length=lowerCamelCase					, **lowerCamelCase					, )

   __A		:  Any       				=		2**8  # utf is 8 bits

   # define special tokens dict
   __A		:  Dict[str, int]       				=		{
       self.pad_token: 0,
       self.bos_token: 1,
       self.eos_token: 2,
       self.mask_token: 3,
       self.cls_token: 4,
       self.sep_token: 5,
   }
   __A		:  Optional[int]       				=		len(self.special_tokens_encoder							)
   __A		:  Dict[int, str]       				=		{v: k for k, v in self.special_tokens_encoder.items()}
 def 		lowercase_( self      :     Dict							):



   """simple docstring"""





   __A		:  Union[str, Any]       				=		self.special_tokens_encoder.copy()
   vocab.update(self.added_tokens_encoder							)
   for i in range(self._utf_vocab_size							):
     __A		:  Optional[int]       				=		chr(lowerCamelCase							)
     __A		:  str       				=		i + len(self.special_tokens_encoder							)
   return vocab
 @property
 def 		lowercase_( self      :     Tuple							):



   """simple docstring"""





   return self._utf_vocab_size + self._num_special_tokens
 def 		lowercase_( self      :     Any					, lowerCamelCase      :     List[int]					, lowerCamelCase      :     Optional[List[int]] = None					, lowerCamelCase      :     bool = False							):



   """simple docstring"""





   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=lowerCamelCase					, token_ids_a=lowerCamelCase					, already_has_special_tokens=lowerCamelCase							)

   # normal case: some special tokens
   if token_ids_a is None:
     return [1] + [0] * len(lowerCamelCase							) + [1]
   return [1] + ([0] * len(lowerCamelCase							)) + [1] + ([0] * len(lowerCamelCase							)) + [1]
 def 		lowercase_( self      :     Optional[int]					, lowerCamelCase      :     List[int]					, lowerCamelCase      :     Optional[List[int]] = None							):



   """simple docstring"""





   if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   else:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]
 def 		lowercase_( self      :     List[str]					, lowerCamelCase      :     str							):



   """simple docstring"""





   __A		:  Dict       				=		[chr(lowerCamelCase							) for i in text.encode("""utf-8"""							)]
   return tokens
 def 		lowercase_( self      :     str					, lowerCamelCase      :     str							):



   """simple docstring"""





   if token in self.special_tokens_encoder:
     __A		:  List[Any]       				=		self.special_tokens_encoder[token]
   elif token in self.added_tokens_encoder:
     __A		:  Dict       				=		self.added_tokens_encoder[token]
   elif len(lowerCamelCase							) != 1:
     __A		:  Optional[int]       				=		self.unk_token_id
   else:
     __A		:  Any       				=		ord(lowerCamelCase							) + self._num_special_tokens
   return token_id
 def 		lowercase_( self      :     Optional[Any]					, lowerCamelCase      :     Any							):



   """simple docstring"""





   if index in self.special_tokens_decoder:
     __A		:  List[str]       				=		self.special_tokens_decoder[index]
   elif index in self.added_tokens_decoder:
     __A		:  str       				=		self.added_tokens_decoder[index]
   else:
     __A		:  Any       				=		chr(index - self._num_special_tokens							)
   return token
 def 		lowercase_( self      :     int					, lowerCamelCase      :     str							):



   """simple docstring"""





   __A		:  Optional[int]       				=		B""""""
   for token in tokens:
     if token in self.special_tokens_decoder:
       __A		:  str       				=		self.special_tokens_decoder[token].encode("""utf-8"""							)
     elif token in self.added_tokens_decoder:
       __A		:  str       				=		self.special_tokens_decoder[token].encode("""utf-8"""							)
     elif token in self.special_tokens_encoder:
       __A		:  List[Any]       				=		token.encode("""utf-8"""							)
     elif token in self.added_tokens_encoder:
       __A		:  str       				=		token.encode("""utf-8"""							)
     else:
       __A		:  Dict       				=		bytes([ord(lowerCamelCase							)]							)
     bstring += tok_string
   __A		:  int       				=		bstring.decode("""utf-8"""					, errors="""replace"""							)
   return string






 def 		lowercase_( self      :     Dict					, lowerCamelCase      :     str					, lowerCamelCase      :     Optional[str] = None							):



   """simple docstring"""





   return ()





'''simple docstring'''
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding







class 			__A					(			_SCREAMING_SNAKE_CASE      ):
 lowerCamelCase												=['''image_processor''', '''tokenizer''']
 lowerCamelCase												='''EfficientNetImageProcessor'''
 lowerCamelCase												=('''BertTokenizer''', '''BertTokenizerFast''')
 def __init__( self      :     str					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Tuple							):



   """simple docstring"""





   super().__init__(lowerCamelCase					, lowerCamelCase							)
 def __call__( self      :     Tuple					, lowerCamelCase      :     str=None					, lowerCamelCase      :     Tuple=None					, lowerCamelCase      :     int="max_length"					, lowerCamelCase      :     int=64					, lowerCamelCase      :     Union[str, Any]=None					, **lowerCamelCase      :     Dict							):



   """simple docstring"""





   if text is None and images is None:
     raise ValueError("""You have to specify either text or images. Both cannot be none."""							)

   if text is not None:
     __A		:  str       				=		self.tokenizer(
         lowerCamelCase					, padding=lowerCamelCase					, max_length=lowerCamelCase					, return_tensors=lowerCamelCase					, **lowerCamelCase							)

   if images is not None:
     __A		:  Optional[Any]       				=		self.image_processor(lowerCamelCase					, return_tensors=lowerCamelCase					, **lowerCamelCase							)

   if text is not None and images is not None:
     __A		:  Optional[int]       				=		image_features.pixel_values
     return encoding
   elif text is not None:
     return encoding
   else:
     return BatchEncoding(data=dict(**lowerCamelCase							)					, tensor_type=lowerCamelCase							)
 def 		lowercase_( self      :     Any					, *lowerCamelCase      :     Optional[Any]					, **lowerCamelCase      :     Union[str, Any]							):



   """simple docstring"""





   return self.tokenizer.batch_decode(*lowerCamelCase					, **lowerCamelCase							)
 def 		lowercase_( self      :     Any					, *lowerCamelCase      :     List[Any]					, **lowerCamelCase      :     Union[str, Any]							):



   """simple docstring"""





   return self.tokenizer.decode(*lowerCamelCase					, **lowerCamelCase							)






 @property
 def 		lowercase_( self      :     int							):



   """simple docstring"""





   __A		:  Dict       				=		self.tokenizer.model_input_names
   __A		:  int       				=		self.image_processor.model_input_names
   return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)



