
'''simple docstring'''







import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
							import os

							from flax.core.frozen_dict import unfreeze
							from flax.traverse_util import flatten_dict

							from transformers import FlaxBertModel

							_lowercase :  List[Any]    							=       "0.12"  # assumed parallelism: 8






@require_flax
@is_staging_test
class  lowerCAmelCase__   (  unittest.TestCase     ):






					@classmethod
					def 	_snake_case    (   cls		):


											"""simple docstring"""





											lowercase_	:     Optional[int]													=       TOKEN
											HfFolder.save_token(__SCREAMING_SNAKE_CASE		)






					@classmethod
					def 	_snake_case    (   cls		):


											"""simple docstring"""





											try:
																	delete_repo(token=cls._token				,							repo_id='''test-model-flax'''		)
											except HTTPError:
																	pass

											try:
																	delete_repo(token=cls._token				,							repo_id='''valid_org/test-model-flax-org'''		)
											except HTTPError:
																	pass






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     List[str]													=       BertConfig(
											    vocab_size=99				,							hidden_size=32				,							num_hidden_layers=5				,							num_attention_heads=4				,							intermediate_size=37		)
											lowercase_	:     Union[str, Any]													=       FlaxBertModel(__SCREAMING_SNAKE_CASE		)
											model.push_to_hub('''test-model-flax'''				,							use_auth_token=self._token		)

											lowercase_	:     Tuple													=       FlaxBertModel.from_pretrained(F'''{USER}/test-model-flax'''		)

											lowercase_	:     List[Any]													=       flatten_dict(unfreeze(model.params		)		)
											lowercase_	:     Tuple													=       flatten_dict(unfreeze(new_model.params		)		)

											for key in base_params.keys():
																	lowercase_	:     Optional[int]													=       (base_params[key] - new_params[key]).sum().item()
																	self.assertLessEqual(__SCREAMING_SNAKE_CASE				,							1E-3				,							msg=F'''{key} not identical'''		)

											# Reset repo
											delete_repo(token=self._token				,							repo_id='''test-model-flax'''		)

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	model.save_pretrained(__SCREAMING_SNAKE_CASE				,							repo_id='''test-model-flax'''				,							push_to_hub=__SCREAMING_SNAKE_CASE				,							use_auth_token=self._token		)

											lowercase_	:     List[str]													=       FlaxBertModel.from_pretrained(F'''{USER}/test-model-flax'''		)

											lowercase_	:     List[str]													=       flatten_dict(unfreeze(model.params		)		)
											lowercase_	:     Dict													=       flatten_dict(unfreeze(new_model.params		)		)

											for key in base_params.keys():
																	lowercase_	:     int													=       (base_params[key] - new_params[key]).sum().item()
																	self.assertLessEqual(__SCREAMING_SNAKE_CASE				,							1E-3				,							msg=F'''{key} not identical'''		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Dict													=       BertConfig(
											    vocab_size=99				,							hidden_size=32				,							num_hidden_layers=5				,							num_attention_heads=4				,							intermediate_size=37		)
											lowercase_	:     Dict													=       FlaxBertModel(__SCREAMING_SNAKE_CASE		)
											model.push_to_hub('''valid_org/test-model-flax-org'''				,							use_auth_token=self._token		)

											lowercase_	:     Dict													=       FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''		)

											lowercase_	:     int													=       flatten_dict(unfreeze(model.params		)		)
											lowercase_	:     Dict													=       flatten_dict(unfreeze(new_model.params		)		)

											for key in base_params.keys():
																	lowercase_	:     List[Any]													=       (base_params[key] - new_params[key]).sum().item()
																	self.assertLessEqual(__SCREAMING_SNAKE_CASE				,							1E-3				,							msg=F'''{key} not identical'''		)

											# Reset repo
											delete_repo(token=self._token				,							repo_id='''valid_org/test-model-flax-org'''		)

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	model.save_pretrained(
																	    __SCREAMING_SNAKE_CASE				,							repo_id='''valid_org/test-model-flax-org'''				,							push_to_hub=__SCREAMING_SNAKE_CASE				,							use_auth_token=self._token		)

											lowercase_	:     Dict													=       FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''		)

											lowercase_	:     List[str]													=       flatten_dict(unfreeze(model.params		)		)
											lowercase_	:     Dict													=       flatten_dict(unfreeze(new_model.params		)		)

											for key in base_params.keys():
																	lowercase_	:     Optional[int]													=       (base_params[key] - new_params[key]).sum().item()
																	self.assertLessEqual(__SCREAMING_SNAKE_CASE				,							1E-3				,							msg=F'''{key} not identical'''		)







def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Dict      ,      __SCREAMING_SNAKE_CASE					: Dict							):



						"""simple docstring"""





						lowercase_	:     int													=       True
						lowercase_	:     str													=       flatten_dict(modela.params							)
						lowercase_	:     List[str]													=       flatten_dict(modela.params							)
						for key in flat_params_a.keys():
												if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]							)							) > 1E-4:
																		lowercase_	:     Any													=       False

						return models_are_equal






@require_flax
class  lowerCAmelCase__   (  unittest.TestCase     ):






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''		)
											lowercase_	:     List[str]													=       FlaxBertModel(__SCREAMING_SNAKE_CASE		)

											lowercase_	:     Union[str, Any]													=       '''bert'''
											with tempfile.TemporaryDirectory() as tmp_dir:
																	model.save_pretrained(os.path.join(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)		)

																	with self.assertRaises(__SCREAMING_SNAKE_CASE		):
																							lowercase_	:     Optional[Any]													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE		)

																	lowercase_	:     int													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE				,							subfolder=__SCREAMING_SNAKE_CASE		)

											self.assertTrue(check_models_equal(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Union[str, Any]													=       BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''		)
											lowercase_	:     Optional[Any]													=       FlaxBertModel(__SCREAMING_SNAKE_CASE		)

											lowercase_	:     List[Any]													=       '''bert'''
											with tempfile.TemporaryDirectory() as tmp_dir:
																	model.save_pretrained(os.path.join(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)				,							max_shard_size='''10KB'''		)

																	with self.assertRaises(__SCREAMING_SNAKE_CASE		):
																							lowercase_	:     List[str]													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE		)

																	lowercase_	:     int													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE				,							subfolder=__SCREAMING_SNAKE_CASE		)

											self.assertTrue(check_models_equal(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     List[str]													=       '''bert'''
											lowercase_	:     Union[str, Any]													=       '''hf-internal-testing/tiny-random-bert-subfolder'''

											with self.assertRaises(__SCREAMING_SNAKE_CASE		):
																	lowercase_	:     Tuple													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE		)

											lowercase_	:     Optional[Any]													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE				,							subfolder=__SCREAMING_SNAKE_CASE		)

											self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     str													=       '''bert'''
											lowercase_	:     Any													=       '''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
											with self.assertRaises(__SCREAMING_SNAKE_CASE		):
																	lowercase_	:     Optional[Any]													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE		)

											lowercase_	:     Tuple													=       FlaxBertModel.from_pretrained(__SCREAMING_SNAKE_CASE				,							subfolder=__SCREAMING_SNAKE_CASE		)

											self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)









"""simple docstring"""


import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 					lowercase(     __a ):







 '''simple docstring'''





 lowercase__ 					=					["image_processor", "tokenizer"]
 lowercase__ 					=					"AutoImageProcessor"
 lowercase__ 					=					"AutoTokenizer"


 def __init__(				self:		List[str],      a_:		List[str]=None,      a_:		Tuple=None,      **a_:		Tuple		):




   '''simple docstring'''



   _snake_case							:		str								=  None
   if "feature_extractor" in kwargs:
     warnings.warn(
         """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
         """ instead.""",      a_,      )
     _snake_case							:		str								=  kwargs.pop("""feature_extractor"""		)

   _snake_case							:		Union[str, Any]								=  image_processor if image_processor is not None else feature_extractor
   if image_processor is None:
     raise ValueError("""You need to specify an `image_processor`."""		)
   if tokenizer is None:
     raise ValueError("""You need to specify a `tokenizer`."""		)

   super().__init__(a_,      a_		)
   _snake_case							:		Dict								=  self.image_processor
   _snake_case							:		Any								=  False


 def __call__(				self:		Any,      *a_:		Any,      **a_:		Tuple		):




   '''simple docstring'''



   if self._in_target_context_manager:
     return self.current_processor(*a_,      **a_		)

   _snake_case							:		Dict								=  kwargs.pop("""images""",      a_		)
   _snake_case							:		Optional[Any]								=  kwargs.pop("""text""",      a_		)
   if len(a_		) > 0:
     _snake_case							:		Optional[int]								=  args[0]
     _snake_case							:		Tuple								=  args[1:]

   if images is None and text is None:
     raise ValueError("""You need to specify either an `images` or `text` input to process."""		)

   if images is not None:
     _snake_case							:		Tuple								=  self.image_processor(a_,      *a_,      **a_		)
   if text is not None:
     _snake_case							:		Tuple								=  self.tokenizer(a_,      **a_		)

   if text is None:
     return inputs
   elif images is None:
     return encodings
   else:
     _snake_case							:		List[str]								=  encodings["""input_ids"""]
     return inputs


 def  UpperCamelCase_   (				self:		Optional[int],      *a_:		Tuple,      **a_:		List[str]		):




   '''simple docstring'''



   return self.tokenizer.batch_decode(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		int,      *a_:		List[str],      **a_:		int		):




   '''simple docstring'''



   return self.tokenizer.decode(*a_,      **a_		)


 @contextmanager
 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   warnings.warn(
       """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
       """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
       """your images inputs, or in a separate call."""		)
   _snake_case							:		Any								=  True
   _snake_case							:		Optional[int]								=  self.tokenizer
   yield
   _snake_case							:		int								=  self.image_processor
   _snake_case							:		Optional[int]								=  False


 def  UpperCamelCase_   (				self:		Dict,      a_:		Optional[Any],      a_:		str=False,      a_:		Optional[Any]=None		):




   '''simple docstring'''



   if added_vocab is None:
     _snake_case							:		Dict								=  self.tokenizer.get_added_vocab()

   _snake_case							:		str								=  {}

   while tokens:
     _snake_case							:		Union[str, Any]								=  re.search(r"""<s_(.*?)>""",      a_,      re.IGNORECASE		)
     if start_token is None:
       break
     _snake_case							:		List[Any]								=  start_token.group(1		)
     _snake_case							:		str								=  re.search(rf"</s_{key}>",      a_,      re.IGNORECASE		)
     _snake_case							:		Dict								=  start_token.group()
     if end_token is None:
       _snake_case							:		List[Any]								=  tokens.replace(a_,      """"""		)
     else:
       _snake_case							:		List[str]								=  end_token.group()
       _snake_case							:		str								=  re.escape(a_		)
       _snake_case							:		str								=  re.escape(a_		)
       _snake_case							:		Union[str, Any]								=  re.search(f"{start_token_escaped}(.*?){end_token_escaped}",      a_,      re.IGNORECASE		)
       if content is not None:
         _snake_case							:		int								=  content.group(1		).strip()
         if r"<s_" in content and r"</s_" in content:  # non-leaf node
           _snake_case							:		List[Any]								=  self.tokenajson(a_,      is_inner_value=a_,      added_vocab=a_		)
           if value:
             if len(a_		) == 1:
               _snake_case							:		List[str]								=  value[0]
             _snake_case							:		List[str]								=  value
         else:  # leaf nodes
           _snake_case							:		Tuple								=  []
           for leaf in content.split(r"""<sep/>"""		):
             _snake_case							:		Tuple								=  leaf.strip()
             if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
               _snake_case							:		int								=  leaf[1:-2]  # for categorical special tokens
             output[key].append(a_		)
           if len(output[key]		) == 1:
             _snake_case							:		int								=  output[key][0]

       _snake_case							:		Any								=  tokens[tokens.find(a_		) + len(a_		) :].strip()
       if tokens[:6] == r"<sep/>":  # non-leaf nodes
         return [output] + self.tokenajson(tokens[6:],      is_inner_value=a_,      added_vocab=a_		)

   if len(a_		):
     return [output] if is_inner_value else output
   else:
     return [] if is_inner_value else {"text_sequence": tokens}


 @property
 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   warnings.warn(
       """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.""",      a_,      )
   return self.image_processor_class


 @property
 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   warnings.warn(
       """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.""",      a_,      )
   return self.image_processor


