import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
   from PIL import Image

   from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
   import torch

if is_tf_available():
   import tensorflow as tf







@require_vision
@require_torchvision
class __snake_case				(						unittest.TestCase		):




   def 	__a						(    self			:				int      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      tempfile.mkdtemp()
     SCREAMING_SNAKE_CASE__										=      SamImageProcessor()
     SCREAMING_SNAKE_CASE__										=      SamProcessor(_lowercase      )
     processor.save_pretrained(self.tmpdirname      )




   def 	__a						(    self			:				Tuple       ,	**_lowercase			:				Optional[int]      ):



     """simple docstring"""






     return AutoProcessor.from_pretrained(self.tmpdirname       ,	**_lowercase      ).image_processor




   def 	__a						(    self			:				List[Any]      ):



     """simple docstring"""






     shutil.rmtree(self.tmpdirname      )




   def 	__a						(    self			:				str      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      [np.random.randint(2_55       ,	size=(3, 30, 4_00)       ,	dtype=np.uinta      )]

     SCREAMING_SNAKE_CASE__										=      [Image.fromarray(np.moveaxis(_lowercase       ,	0       ,	-1      )      ) for x in image_inputs]

     return image_inputs




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=self.get_image_processor()      )
     processor.save_pretrained(self.tmpdirname      )

     SCREAMING_SNAKE_CASE__										=      self.get_image_processor(do_normalize=_lowercase       ,	padding_value=1.0      )

     SCREAMING_SNAKE_CASE__										=      SamProcessor.from_pretrained(self.tmpdirname       ,	do_normalize=_lowercase       ,	padding_value=1.0      )

     self.assertEqual(processor.image_processor.to_json_string()       ,	image_processor_add_kwargs.to_json_string()      )
     self.assertIsInstance(processor.image_processor       ,	_lowercase      )




   def 	__a						(    self			:				int      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )

     SCREAMING_SNAKE_CASE__										=      self.prepare_image_inputs()

     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""np"""      )
     SCREAMING_SNAKE_CASE__										=      processor(images=_lowercase       ,	return_tensors="""np"""      )

     input_feat_extract.pop("""original_sizes"""      )  # pop original_sizes as it is popped in the processor
     input_feat_extract.pop("""reshaped_input_sizes"""      )  # pop original_sizes as it is popped in the processor

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()       ,	input_processor[key].sum()       ,	delta=1E-2      )




   @require_torch
   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )
     SCREAMING_SNAKE_CASE__										=      [torch.ones((1, 3, 5, 5)      )]

     SCREAMING_SNAKE_CASE__										=      [[17_64, 26_46]]

     SCREAMING_SNAKE_CASE__										=      [[6_83, 10_24]]
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(_lowercase       ,	_lowercase       ,	_lowercase      )
     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
         _lowercase       ,	torch.tensor(_lowercase      )       ,	torch.tensor(_lowercase      )      )
     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     # should also work with np
     SCREAMING_SNAKE_CASE__										=      [np.ones((1, 3, 5, 5)      )]
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(_lowercase       ,	np.array(_lowercase      )       ,	np.array(_lowercase      )      )

     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     SCREAMING_SNAKE_CASE__										=      [[1, 0], [0, 1]]
     with self.assertRaises(_lowercase      ):
       SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(_lowercase       ,	np.array(_lowercase      )       ,	np.array(_lowercase      )      )







@require_vision
@require_tf
class __snake_case				(						unittest.TestCase		):




   def 	__a						(    self			:				str      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      tempfile.mkdtemp()
     SCREAMING_SNAKE_CASE__										=      SamImageProcessor()
     SCREAMING_SNAKE_CASE__										=      SamProcessor(_lowercase      )
     processor.save_pretrained(self.tmpdirname      )




   def 	__a						(    self			:				Optional[int]       ,	**_lowercase			:				Dict      ):



     """simple docstring"""






     return AutoProcessor.from_pretrained(self.tmpdirname       ,	**_lowercase      ).image_processor




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     shutil.rmtree(self.tmpdirname      )




   def 	__a						(    self			:				Any      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      [np.random.randint(2_55       ,	size=(3, 30, 4_00)       ,	dtype=np.uinta      )]

     SCREAMING_SNAKE_CASE__										=      [Image.fromarray(np.moveaxis(_lowercase       ,	0       ,	-1      )      ) for x in image_inputs]

     return image_inputs




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=self.get_image_processor()      )
     processor.save_pretrained(self.tmpdirname      )

     SCREAMING_SNAKE_CASE__										=      self.get_image_processor(do_normalize=_lowercase       ,	padding_value=1.0      )

     SCREAMING_SNAKE_CASE__										=      SamProcessor.from_pretrained(self.tmpdirname       ,	do_normalize=_lowercase       ,	padding_value=1.0      )

     self.assertEqual(processor.image_processor.to_json_string()       ,	image_processor_add_kwargs.to_json_string()      )
     self.assertIsInstance(processor.image_processor       ,	_lowercase      )




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )

     SCREAMING_SNAKE_CASE__										=      self.prepare_image_inputs()

     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""np"""      )
     SCREAMING_SNAKE_CASE__										=      processor(images=_lowercase       ,	return_tensors="""np"""      )

     input_feat_extract.pop("""original_sizes"""      )  # pop original_sizes as it is popped in the processor
     input_feat_extract.pop("""reshaped_input_sizes"""      )  # pop reshaped_input_sizes as it is popped in the processor

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()       ,	input_processor[key].sum()       ,	delta=1E-2      )




   @require_tf
   def 	__a						(    self			:				Dict      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )
     SCREAMING_SNAKE_CASE__										=      [tf.ones((1, 3, 5, 5)      )]

     SCREAMING_SNAKE_CASE__										=      [[17_64, 26_46]]

     SCREAMING_SNAKE_CASE__										=      [[6_83, 10_24]]
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(_lowercase       ,	_lowercase       ,	_lowercase       ,	return_tensors="""tf"""      )
     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
         _lowercase       ,	tf.convert_to_tensor(_lowercase      )       ,	tf.convert_to_tensor(_lowercase      )       ,	return_tensors="""tf"""       ,	)
     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     # should also work with np
     SCREAMING_SNAKE_CASE__										=      [np.ones((1, 3, 5, 5)      )]
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
         _lowercase       ,	np.array(_lowercase      )       ,	np.array(_lowercase      )       ,	return_tensors="""tf"""      )

     self.assertEqual(masks[0].shape       ,	(1, 3, 17_64, 26_46)      )

     SCREAMING_SNAKE_CASE__										=      [[1, 0], [0, 1]]
     with self.assertRaises(tf.errors.InvalidArgumentError      ):
       SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
           _lowercase       ,	np.array(_lowercase      )       ,	np.array(_lowercase      )       ,	return_tensors="""tf"""      )







@require_vision
@require_torchvision
class __snake_case				(						unittest.TestCase		):




   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      tempfile.mkdtemp()
     SCREAMING_SNAKE_CASE__										=      SamImageProcessor()
     SCREAMING_SNAKE_CASE__										=      SamProcessor(_lowercase      )
     processor.save_pretrained(self.tmpdirname      )




   def 	__a						(    self			:				List[Any]       ,	**_lowercase			:				Tuple      ):



     """simple docstring"""






     return AutoProcessor.from_pretrained(self.tmpdirname       ,	**_lowercase      ).image_processor




   def 	__a						(    self			:				Union[str, Any]      ):



     """simple docstring"""






     shutil.rmtree(self.tmpdirname      )




   def 	__a						(    self			:				Optional[Any]      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      [np.random.randint(2_55       ,	size=(3, 30, 4_00)       ,	dtype=np.uinta      )]

     SCREAMING_SNAKE_CASE__										=      [Image.fromarray(np.moveaxis(_lowercase       ,	0       ,	-1      )      ) for x in image_inputs]

     return image_inputs




   @is_pt_tf_cross_test
   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )
     SCREAMING_SNAKE_CASE__										=      np.random.randint(0       ,	2       ,	size=(1, 3, 5, 5)      ).astype(np.floataa      )
     SCREAMING_SNAKE_CASE__										=      [tf.convert_to_tensor(_lowercase      )]
     SCREAMING_SNAKE_CASE__										=      [torch.tensor(_lowercase      )]

     SCREAMING_SNAKE_CASE__										=      [[17_64, 26_46]]

     SCREAMING_SNAKE_CASE__										=      [[6_83, 10_24]]
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
         _lowercase       ,	_lowercase       ,	_lowercase       ,	return_tensors="""tf"""      )
     SCREAMING_SNAKE_CASE__										=      processor.post_process_masks(
         _lowercase       ,	_lowercase       ,	_lowercase       ,	return_tensors="""pt"""      )

     self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()      )      )




   @is_pt_tf_cross_test
   def 	__a						(    self			:				Tuple      ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__										=      self.get_image_processor()

     SCREAMING_SNAKE_CASE__										=      SamProcessor(image_processor=_lowercase      )

     SCREAMING_SNAKE_CASE__										=      self.prepare_image_inputs()

     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""pt"""      )['''pixel_values'''].numpy()
     SCREAMING_SNAKE_CASE__										=      processor(images=_lowercase       ,	return_tensors="""pt"""      )['''pixel_values'''].numpy()

     SCREAMING_SNAKE_CASE__										=      image_processor(_lowercase       ,	return_tensors="""tf"""      )['''pixel_values'''].numpy()
     SCREAMING_SNAKE_CASE__										=      processor(images=_lowercase       ,	return_tensors="""tf"""      )['''pixel_values'''].numpy()

     self.assertTrue(np.allclose(_lowercase       ,	_lowercase      )      )
     self.assertTrue(np.allclose(_lowercase       ,	_lowercase      )      )
     self.assertTrue(np.allclose(_lowercase       ,	_lowercase      )      )


'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_rembert import RemBertTokenizer
else:
      lowercase			:				Optional[Any]    		=			None

lowercase			:				Tuple    		=			logging.get_logger(__name__)
lowercase			:				Dict    		=			{'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

lowercase			:				Tuple    		=			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

lowercase			:				List[str]    		=			{
    'google/rembert': 2_56,
}

lowercase			:				Dict    		=			'▁'





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		RemBertTokenizer





     def __init__(							self						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<unk>"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<pad>"						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[MASK]"						,			**SCREAMING_SNAKE_CASE						,			) ->				List[str]:






         """simple docstring"""






         A       :   Optional[Any]					=						AddedToken(SCREAMING_SNAKE_CASE						,			lstrip=SCREAMING_SNAKE_CASE						,			rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	) else mask_token

         super().__init__(
             SCREAMING_SNAKE_CASE						,			tokenizer_file=SCREAMING_SNAKE_CASE						,			do_lower_case=SCREAMING_SNAKE_CASE						,			remove_space=SCREAMING_SNAKE_CASE						,			keep_accents=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			sep_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			cls_token=SCREAMING_SNAKE_CASE						,			mask_token=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   List[Any]					=						do_lower_case
         A       :   str					=						remove_space
         A       :   int					=						keep_accents
         A       :   Union[str, Any]					=						vocab_file
         A       :   List[Any]					=						False if not self.vocab_file else True





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   List[Any]					=						[self.sep_token_id]
         A       :   Tuple					=						[self.cls_token_id]
         if token_ids_a is None:
             return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None						,			SCREAMING_SNAKE_CASE = False	) ->				List[int]:






         """simple docstring"""






         if already_has_special_tokens:
             if token_ids_a is not None:
                 raise ValueError(
                     '''You should not supply a second sequence if the provided sequence of '''
                     '''ids is already formatted with special tokens for the model.'''	)
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
             return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   Tuple					=						[self.sep_token_id]
         A       :   Dict					=						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         if not os.path.isdir(SCREAMING_SNAKE_CASE	):
             logger.error('''Vocabulary path ({}) should be a directory'''.format(SCREAMING_SNAKE_CASE	)	)
             return
         A       :   Any					=						os.path.join(
             SCREAMING_SNAKE_CASE						,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	):
             copyfile(self.vocab_file						,			SCREAMING_SNAKE_CASE	)

         return (out_vocab_file,)




