



'''simple docstring'''






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__:   int								=			get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
class 				SCREAMING_SNAKE_CASE(							A__							,		unittest.TestCase			):







	"""simple docstring"""






	lowerCamelCase__			 =       XLMProphetNetTokenizer
	lowerCamelCase__			 =       False
	lowerCamelCase__			 =       True







	def      A	(						self			:     Dict		)			->		Any:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase :       Dict 							=		XLMProphetNetTokenizer(__snake_case		,			keep_accents=__snake_case		)
								tokenizer.save_pretrained(self.tmpdirname		)







	def      A	(						self			:     Any		)			->		Tuple:
								UpperCAmelCase :       Any 							=		'''[PAD]'''
								UpperCAmelCase :       Optional[Any] 							=		0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case		)		,			__snake_case		)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case		)		,			__snake_case		)







	def      A	(						self			:     Optional[Any]		)			->		Any:
								UpperCAmelCase :       Union[str, Any] 							=		list(self.get_tokenizer().get_vocab().keys()		)

								self.assertEqual(vocab_keys[0]		,			'''[PAD]'''		)
								self.assertEqual(vocab_keys[1]		,			'''[CLS]'''		)
								self.assertEqual(vocab_keys[-1]		,			'''j'''		)
								self.assertEqual(len(__snake_case		)		,			1012		)







	def      A	(						self			:     int		)			->		Any:
								self.assertEqual(self.get_tokenizer().vocab_size		,			1012		)







	def      A	(						self			:     Tuple		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		XLMProphetNetTokenizer(__snake_case		,			keep_accents=__snake_case		)

								UpperCAmelCase :       List[Any] 							=		tokenizer.tokenize('''This is a test'''		)
								self.assertListEqual(__snake_case		,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(__snake_case		)		,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]		,			)

								UpperCAmelCase :       Optional[int] 							=		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
								self.assertListEqual(
								    __snake_case		,			[
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]		,			)
								UpperCAmelCase :       Union[str, Any] 							=		tokenizer.convert_tokens_to_ids(__snake_case		)
								self.assertListEqual(
								    __snake_case		,			[
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
								    ]		,			)

								UpperCAmelCase :       Tuple 							=		tokenizer.convert_ids_to_tokens(__snake_case		)
								self.assertListEqual(
								    __snake_case		,			[
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''[UNK]''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''[UNK]''',
								        '''.''',
								    ]		,			)







	@cached_property
	def      A	(						self			:     str		)			->		List[Any]:
								return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''		)







	@slow
	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       int 							=		'''Hello World!'''
								UpperCAmelCase :       str 							=		[35389, 6672, 49, 2]
								self.assertListEqual(__snake_case		,			self.big_tokenizer.encode(__snake_case		)		)







	@slow
	def      A	(						self			:     Dict		)			->		Union[str, Any]:
								# fmt: off
								UpperCAmelCase :       int 							=		{'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=__snake_case		,			model_name='''microsoft/xprophetnet-large-wiki100-cased'''		,			revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''		,			)










'''simple docstring'''






import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
						import os

						from flax.core.frozen_dict import unfreeze
						from flax.traverse_util import flatten_dict

						from transformers import FlaxBertModel

						UpperCamelCase__:   Tuple								=			"0.12"  # assumed parallelism: 8

@require_flax
@is_staging_test
class 				SCREAMING_SNAKE_CASE(							unittest.TestCase			):







	"""simple docstring"""






	@classmethod
	def      A	(						cls			:     Union[str, Any]		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		TOKEN
								HfFolder.save_token(__snake_case		)







	@classmethod
	def      A	(						cls			:     List[str]		)			->		Tuple:
								try:
															delete_repo(token=cls._token		,			repo_id='''test-model-flax'''		)
								except HTTPError:
															pass

								try:
															delete_repo(token=cls._token		,			repo_id='''valid_org/test-model-flax-org'''		)
								except HTTPError:
															pass







	def      A	(						self			:     int		)			->		Tuple:
								UpperCAmelCase :       List[Any] 							=		BertConfig(
								    vocab_size=99		,			hidden_size=32		,			num_hidden_layers=5		,			num_attention_heads=4		,			intermediate_size=37		)
								UpperCAmelCase :       Dict 							=		FlaxBertModel(__snake_case		)
								model.push_to_hub('''test-model-flax'''		,			use_auth_token=self._token		)

								UpperCAmelCase :       Tuple 							=		FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""		)

								UpperCAmelCase :       List[Any] 							=		flatten_dict(unfreeze(model.params		)		)
								UpperCAmelCase :       Tuple 							=		flatten_dict(unfreeze(new_model.params		)		)

								for key in base_params.keys():
															UpperCAmelCase :       Union[str, Any] 							=		(base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(__snake_case		,			1E-3		,			msg=F"""{key} not identical"""		)

								# Reset repo
								delete_repo(token=self._token		,			repo_id='''test-model-flax'''		)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(__snake_case		,			repo_id='''test-model-flax'''		,			push_to_hub=__snake_case		,			use_auth_token=self._token		)

								UpperCAmelCase :       str 							=		FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""		)

								UpperCAmelCase :       Any 							=		flatten_dict(unfreeze(model.params		)		)
								UpperCAmelCase :       str 							=		flatten_dict(unfreeze(new_model.params		)		)

								for key in base_params.keys():
															UpperCAmelCase :       Optional[Any] 							=		(base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(__snake_case		,			1E-3		,			msg=F"""{key} not identical"""		)







	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       Dict 							=		BertConfig(
								    vocab_size=99		,			hidden_size=32		,			num_hidden_layers=5		,			num_attention_heads=4		,			intermediate_size=37		)
								UpperCAmelCase :       Optional[Any] 							=		FlaxBertModel(__snake_case		)
								model.push_to_hub('''valid_org/test-model-flax-org'''		,			use_auth_token=self._token		)

								UpperCAmelCase :       Union[str, Any] 							=		FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''		)

								UpperCAmelCase :       List[Any] 							=		flatten_dict(unfreeze(model.params		)		)
								UpperCAmelCase :       int 							=		flatten_dict(unfreeze(new_model.params		)		)

								for key in base_params.keys():
															UpperCAmelCase :       Any 							=		(base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(__snake_case		,			1E-3		,			msg=F"""{key} not identical"""		)

								# Reset repo
								delete_repo(token=self._token		,			repo_id='''valid_org/test-model-flax-org'''		)

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(
															    __snake_case		,			repo_id='''valid_org/test-model-flax-org'''		,			push_to_hub=__snake_case		,			use_auth_token=self._token		)

								UpperCAmelCase :       str 							=		FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''		)

								UpperCAmelCase :       Any 							=		flatten_dict(unfreeze(model.params		)		)
								UpperCAmelCase :       Optional[Any] 							=		flatten_dict(unfreeze(new_model.params		)		)

								for key in base_params.keys():
															UpperCAmelCase :       int 							=		(base_params[key] - new_params[key]).sum().item()
															self.assertLessEqual(__snake_case		,			1E-3		,			msg=F"""{key} not identical"""		)


def  snake_case_     (					_lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						Any  )      ->   Union[str, Any]:
							UpperCAmelCase :       str 							=		True
							UpperCAmelCase :       int 							=		flatten_dict(modela.params  )
							UpperCAmelCase :       Dict 							=		flatten_dict(modela.params  )
							for key in flat_params_a.keys():
														if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]  )  ) > 1e-4:
																					UpperCAmelCase :       Dict 							=		False

							return models_are_equal




@require_flax
class 				SCREAMING_SNAKE_CASE(							unittest.TestCase			):







	"""simple docstring"""






	def      A	(						self			:     Tuple		)			->		Union[str, Any]:
								UpperCAmelCase :       List[Any] 							=		BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''		)
								UpperCAmelCase :       Dict 							=		FlaxBertModel(__snake_case		)

								UpperCAmelCase :       int 							=		'''bert'''
								with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(os.path.join(__snake_case		,			__snake_case		)		)

															with self.assertRaises(__snake_case		):
																						UpperCAmelCase :       Tuple 							=		FlaxBertModel.from_pretrained(__snake_case		)

															UpperCAmelCase :       str 							=		FlaxBertModel.from_pretrained(__snake_case		,			subfolder=__snake_case		)

								self.assertTrue(check_models_equal(__snake_case		,			__snake_case		)		)







	def      A	(						self			:     List[str]		)			->		Dict:
								UpperCAmelCase :       Dict 							=		BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''		)
								UpperCAmelCase :       Dict 							=		FlaxBertModel(__snake_case		)

								UpperCAmelCase :       Optional[int] 							=		'''bert'''
								with tempfile.TemporaryDirectory() as tmp_dir:
															model.save_pretrained(os.path.join(__snake_case		,			__snake_case		)		,			max_shard_size='''10KB'''		)

															with self.assertRaises(__snake_case		):
																						UpperCAmelCase :       Any 							=		FlaxBertModel.from_pretrained(__snake_case		)

															UpperCAmelCase :       Union[str, Any] 							=		FlaxBertModel.from_pretrained(__snake_case		,			subfolder=__snake_case		)

								self.assertTrue(check_models_equal(__snake_case		,			__snake_case		)		)







	def      A	(						self			:     Optional[int]		)			->		str:
								UpperCAmelCase :       Dict 							=		'''bert'''
								UpperCAmelCase :       int 							=		'''hf-internal-testing/tiny-random-bert-subfolder'''

								with self.assertRaises(__snake_case		):
															UpperCAmelCase :       Optional[Any] 							=		FlaxBertModel.from_pretrained(__snake_case		)

								UpperCAmelCase :       Tuple 							=		FlaxBertModel.from_pretrained(__snake_case		,			subfolder=__snake_case		)

								self.assertIsNotNone(__snake_case		)







	def      A	(						self			:     Dict		)			->		List[Any]:
								UpperCAmelCase :       Optional[int] 							=		'''bert'''
								UpperCAmelCase :       int 							=		'''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
								with self.assertRaises(__snake_case		):
															UpperCAmelCase :       Dict 							=		FlaxBertModel.from_pretrained(__snake_case		)

								UpperCAmelCase :       Union[str, Any] 							=		FlaxBertModel.from_pretrained(__snake_case		,			subfolder=__snake_case		)

								self.assertIsNotNone(__snake_case		)





