

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_a		       =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_a		       =   {
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

_a		       =   {
    '''google/electra-small-generator''': 5_1_2,
    '''google/electra-base-generator''': 5_1_2,
    '''google/electra-large-generator''': 5_1_2,
    '''google/electra-small-discriminator''': 5_1_2,
    '''google/electra-base-discriminator''': 5_1_2,
    '''google/electra-large-discriminator''': 5_1_2,
}

_a		       =   {
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}






class 							A_ (		_lowercase					):
     _lowercase			:     Union[str, Any]								=	VOCAB_FILES_NAMES
     _lowercase			:     Tuple								=	PRETRAINED_VOCAB_FILES_MAP
     _lowercase			:     List[Any]								=	PRETRAINED_INIT_CONFIGURATION
     _lowercase			:     List[str]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowercase			:     Optional[Any]								=	ElectraTokenizer


     def __init__(							self			:		int			,  UpperCAmelCase			:		Optional[Any]=None			,  UpperCAmelCase			:		List[Any]=None			,  UpperCAmelCase			:		str=True			,  UpperCAmelCase			:		Tuple="[UNK]"			,  UpperCAmelCase			:		Any="[SEP]"			,  UpperCAmelCase			:		Tuple="[PAD]"			,  UpperCAmelCase			:		Dict="[CLS]"			,  UpperCAmelCase			:		int="[MASK]"			,  UpperCAmelCase			:		Optional[int]=True			,  UpperCAmelCase			:		Tuple=None			,  **UpperCAmelCase			:		Optional[int]			,  )							->	Union[str, Any]:
      super().__init__(
          UpperCamelCase__			,  tokenizer_file=UpperCamelCase__			,  do_lower_case=UpperCamelCase__			,  unk_token=UpperCamelCase__			,  sep_token=UpperCamelCase__			,  pad_token=UpperCamelCase__			,  cls_token=UpperCamelCase__			,  mask_token=UpperCamelCase__			,  tokenize_chinese_chars=UpperCamelCase__			,  strip_accents=UpperCamelCase__			,  **UpperCamelCase__			,  )

      __lowerCAmelCase:   List[Any]   	=      json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
      if (
          normalizer_state.get('lowercase'			,  UpperCamelCase__					) != do_lower_case
          or normalizer_state.get('strip_accents'			,  UpperCamelCase__					) != strip_accents
          or normalizer_state.get('handle_chinese_chars'			,  UpperCamelCase__					) != tokenize_chinese_chars
      ):
       __lowerCAmelCase:   Optional[Any]   	=      getattr(UpperCamelCase__			,  normalizer_state.pop('type'					)					)
       __lowerCAmelCase:   Optional[int]   	=      do_lower_case
       __lowerCAmelCase:   List[Any]   	=      strip_accents
       __lowerCAmelCase:   Tuple   	=      tokenize_chinese_chars
       __lowerCAmelCase:   Dict   	=      normalizer_class(**UpperCamelCase__					)

      __lowerCAmelCase:   Optional[Any]   	=      do_lower_case


     def   UpperCAmelCase  (							self			:		List[Any]			,  UpperCAmelCase			:		Optional[Any]			,  UpperCAmelCase			:		Any=None					)							->	List[Any]:
      __lowerCAmelCase:   Any   	=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output


     def   UpperCAmelCase  (							self			:		Any			,  UpperCAmelCase			:		List[int]			,  UpperCAmelCase			:		Optional[List[int]] = None					)							->	Any:
      __lowerCAmelCase:   List[Any]   	=      [self.sep_token_id]
      __lowerCAmelCase:   List[Any]   	=      [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep					) * [0]
      return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]


     def   UpperCAmelCase  (							self			:		List[Any]			,  UpperCAmelCase			:		str			,  UpperCAmelCase			:		Optional[str] = None					)							->	Tuple:
      __lowerCAmelCase:   int   	=      self._tokenizer.model.save(UpperCamelCase__			,  name=UpperCamelCase__					)
      return tuple(UpperCamelCase__					)













'''simple docstring'''







import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
			from PIL import Image

			from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor

@require_tokenizers
@require_vision
class _lowercase	(				unittest.TestCase		):




							def 		lowerCamelCase_				(							self:				Dict     ):
														lowerCamelCase__					:	int         =       tempfile.mkdtemp()

														# fmt: off
														lowerCamelCase__					:	int         =       ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing""", """,""", """low""", """lowest"""]
														# fmt: on
														lowerCamelCase__					:	Union[str, Any]         =       os.path.join(self.tmpdirname			,							VOCAB_FILES_NAMES["""vocab_file"""]     )
														with open(self.vocab_file			,							"""w"""			,							encoding="""utf-8"""     ) as vocab_writer:
																					vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )

														lowerCamelCase__					:	Tuple         =       {
														    """do_resize""": True,
														    """size""": {"""height""": 18, """width""": 18},
														    """do_normalize""": True,
														    """image_mean""": [0.5, 0.5, 0.5],
														    """image_std""": [0.5, 0.5, 0.5],
														}
														lowerCamelCase__					:	Tuple         =       os.path.join(self.tmpdirname			,							UpperCamelCase__     )
														with open(self.image_processor_file			,							"""w"""			,							encoding="""utf-8"""     ) as fp:
																					json.dump(UpperCamelCase__			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				str			,							**UpperCamelCase__:				List[str]     ):
														return BertTokenizer.from_pretrained(self.tmpdirname			,							**UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				int			,							**UpperCamelCase__:				Tuple     ):
														return ViTImageProcessor.from_pretrained(self.tmpdirname			,							**UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[Any]     ):
														shutil.rmtree(self.tmpdirname     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	List[Any]         =       [np.random.randint(255			,							size=(3, 30, 400)			,							dtype=np.uinta     )]

														lowerCamelCase__					:	Tuple         =       [Image.fromarray(np.moveaxis(UpperCamelCase__			,							0			,							-1     )     ) for x in image_inputs]

														return image_inputs




							def 		lowerCamelCase_				(							self:				int     ):
														lowerCamelCase__					:	Optional[Any]         =       self.get_tokenizer()
														lowerCamelCase__					:	Dict         =       self.get_image_processor()

														lowerCamelCase__					:	Optional[Any]         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														processor.save_pretrained(self.tmpdirname     )
														lowerCamelCase__					:	int         =       VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname     )

														self.assertEqual(processor.tokenizer.get_vocab()			,							tokenizer.get_vocab()     )
														self.assertIsInstance(processor.tokenizer			,							(BertTokenizer, BertTokenizerFast)     )

														self.assertEqual(processor.image_processor.to_json_string()			,							image_processor.to_json_string()     )
														self.assertIsInstance(processor.image_processor			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Dict         =       VisionTextDualEncoderProcessor(
														    tokenizer=self.get_tokenizer()			,							image_processor=self.get_image_processor()     )
														processor.save_pretrained(self.tmpdirname     )

														lowerCamelCase__					:	int         =       self.get_tokenizer(bos_token="""(BOS)"""			,							eos_token="""(EOS)"""     )
														lowerCamelCase__					:	List[Any]         =       self.get_image_processor(do_normalize=UpperCamelCase__			,							padding_value=1.0     )

														lowerCamelCase__					:	Tuple         =       VisionTextDualEncoderProcessor.from_pretrained(
														    self.tmpdirname			,							bos_token="""(BOS)"""			,							eos_token="""(EOS)"""			,							do_normalize=UpperCamelCase__			,							padding_value=1.0     )

														self.assertEqual(processor.tokenizer.get_vocab()			,							tokenizer_add_kwargs.get_vocab()     )
														self.assertIsInstance(processor.tokenizer			,							(BertTokenizer, BertTokenizerFast)     )

														self.assertEqual(processor.image_processor.to_json_string()			,							image_processor_add_kwargs.to_json_string()     )
														self.assertIsInstance(processor.image_processor			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														lowerCamelCase__					:	Optional[Any]         =       self.get_image_processor()
														lowerCamelCase__					:	Union[str, Any]         =       self.get_tokenizer()

														lowerCamelCase__					:	Any         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														lowerCamelCase__					:	List[Any]         =       self.prepare_image_inputs()

														lowerCamelCase__					:	List[str]         =       image_processor(UpperCamelCase__			,							return_tensors="""np"""     )
														lowerCamelCase__					:	Optional[Any]         =       processor(images=UpperCamelCase__			,							return_tensors="""np"""     )

														for key in input_feat_extract.keys():
																					self.assertAlmostEqual(input_feat_extract[key].sum()			,							input_processor[key].sum()			,							delta=1e-2     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	Any         =       self.get_image_processor()
														lowerCamelCase__					:	List[str]         =       self.get_tokenizer()

														lowerCamelCase__					:	List[Any]         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														lowerCamelCase__					:	List[Any]         =       """lower newer"""

														lowerCamelCase__					:	Union[str, Any]         =       processor(text=UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       tokenizer(UpperCamelCase__     )

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]			,							encoded_processor[key]     )




							def 		lowerCamelCase_				(							self:				Dict     ):
														lowerCamelCase__					:	Optional[Any]         =       self.get_image_processor()
														lowerCamelCase__					:	List[Any]         =       self.get_tokenizer()

														lowerCamelCase__					:	List[Any]         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       """lower newer"""
														lowerCamelCase__					:	Dict         =       self.prepare_image_inputs()

														lowerCamelCase__					:	Tuple         =       processor(text=UpperCamelCase__			,							images=UpperCamelCase__     )

														self.assertListEqual(list(inputs.keys()     )			,							["""input_ids""", """token_type_ids""", """attention_mask""", """pixel_values"""]     )

														# test if it raises when no input is passed
														with self.assertRaises(UpperCamelCase__     ):
																					processor()




							def 		lowerCamelCase_				(							self:				int     ):
														lowerCamelCase__					:	List[str]         =       self.get_image_processor()
														lowerCamelCase__					:	List[str]         =       self.get_tokenizer()

														lowerCamelCase__					:	int         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														lowerCamelCase__					:	Dict         =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														lowerCamelCase__					:	Union[str, Any]         =       processor.batch_decode(UpperCamelCase__     )
														lowerCamelCase__					:	Dict         =       tokenizer.batch_decode(UpperCamelCase__     )

														self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				List[str]     ):
														lowerCamelCase__					:	Any         =       self.get_image_processor()
														lowerCamelCase__					:	Union[str, Any]         =       self.get_tokenizer()

														lowerCamelCase__					:	int         =       VisionTextDualEncoderProcessor(tokenizer=UpperCamelCase__			,							image_processor=UpperCamelCase__     )

														lowerCamelCase__					:	Optional[Any]         =       """lower newer"""
														lowerCamelCase__					:	str         =       self.prepare_image_inputs()

														lowerCamelCase__					:	int         =       processor(text=UpperCamelCase__			,							images=UpperCamelCase__     )

														self.assertListEqual(list(inputs.keys()     )			,							processor.model_input_names     )

