



"""simple docstring"""


from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

snake_case_								=			{
    name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
}

snake_case_								=			datasets.load_dataset("""xnli""", split="""test+validation""")

snake_case_								=			0
snake_case_								=			0
snake_case_								=			0
snake_case_								=			0

def 					_lowerCAmelCase (  lowercase_						,      lowercase_						,      lowercase_						,      lowercase_		):
						if spm_diff == list(reversed(UpperCAmelCase_		)		):
												# AAA -> AA+A vs A+AA case.
												return True
						elif len(UpperCAmelCase_		) == len(UpperCAmelCase_		) and fast.decode(UpperCAmelCase_		) == fast.decode(UpperCAmelCase_		):
												# Second order OK
												# Barrich -> Barr + ich vs Bar + rich
												return True
						UpperCAmelCase			 =					slow.encode(slow.decode(UpperCAmelCase_		)		)
						UpperCAmelCase			 =					fast.encode(fast.decode(UpperCAmelCase_		)		)
						if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
												# Type 3 error.
												# Snehagatha ->
												#       Sne, h, aga, th, a
												#       Sne, ha, gat, ha
												# Encoding the wrong with sp does not even recover what spm gave us
												# It fits tokenizer however...
												return True
						return False

def 					_lowerCAmelCase (  lowercase_						,      lowercase_						,      lowercase_		):
						UpperCAmelCase			 =					fast.encode_plus(UpperCAmelCase_		)[0]
						UpperCAmelCase			 =					enc.offsets
						UpperCAmelCase ,       UpperCAmelCase			 =					offsets[idx], offsets[idx - 1]
						if curr is not None and line[curr[0] : curr[1]] == "\u200f":
												return True
						if prev is not None and line[prev[0] : prev[1]] == "\u200f":
												return True

def 					_lowerCAmelCase (  lowercase_						,      lowercase_						,      lowercase_						,      lowercase_						,      lowercase_		):
						# Encoding can be the same with same result AAA -> A + AA vs AA + A
						# We can check that we use at least exactly the same number of tokens.
						for i, (spm_id, tok_id) in enumerate(zip(UpperCAmelCase_						,      UpperCAmelCase_		)		):
												if spm_id != tok_id:
																		break
						UpperCAmelCase			 =					i
						for i, (spm_id, tok_id) in enumerate(zip(reversed(UpperCAmelCase_		)						,      reversed(UpperCAmelCase_		)		)		):
												if spm_id != tok_id:
																		break
						UpperCAmelCase			 =					len(UpperCAmelCase_		) - i

						UpperCAmelCase			 =					spm_ids[first:last]
						UpperCAmelCase			 =					tok_ids[first:last]

						if check_diff(UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_		):
												return True

						if check_LTR_mark(UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_		):
												return True

						if last - first > 5:
												# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
												UpperCAmelCase			 =					Counter(spm_ids[first:last]		)
												UpperCAmelCase			 =					Counter(tok_ids[first:last]		)

												UpperCAmelCase			 =					{spm_ for (spm_, si) in spms.items() if toks.get(spm_						,      0		) == si}
												UpperCAmelCase			 =					3
												for i in range(last - first - min_width		):
																		if all(spm_ids[first + i + j] in removable_tokens for j in range(UpperCAmelCase_		)		):
																								UpperCAmelCase			 =					[
																								    k
																								    for k in range(last - first - min_width		)
																								    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																								]
																								for j in possible_matches:
																														if check_diff(spm_ids[first : first + i]						,      tok_ids[first : first + j]						,      UpperCAmelCase_						,      UpperCAmelCase_		) and check_details(
																														    UpperCAmelCase_						,      spm_ids[first + i : last]						,      tok_ids[first + j : last]						,      UpperCAmelCase_						,      UpperCAmelCase_						,      ):
																																				return True

						print(F"""Spm: {[fast.decode([spm_ids[i]]		) for i in range(UpperCAmelCase_						,      UpperCAmelCase_		)]}"""		)
						try:
												print(F"""Tok: {[fast.decode([tok_ids[i]]		) for i in range(UpperCAmelCase_						,      UpperCAmelCase_		)]}"""		)
						except Exception:
												pass

						UpperCAmelCase			 =					fast.decode(spm_ids[:first]		)
						UpperCAmelCase			 =					fast.decode(spm_ids[last:]		)
						UpperCAmelCase			 =					fast.decode(spm_ids[first:last]		)
						print()
						print(UpperCAmelCase_		)
						return False

def 					_lowerCAmelCase (  lowercase_						,      lowercase_						,      lowercase_		):
						global perfect
						global imperfect
						global wrong
						global total

						UpperCAmelCase			 =					slow.encode(UpperCAmelCase_		)
						UpperCAmelCase			 =					fast.encode(UpperCAmelCase_		)

						UpperCAmelCase			 =					False
						total += 1

						if slow_ids != fast_ids:
												if check_details(UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_		):
																		UpperCAmelCase			 =					True
																		imperfect += 1
												else:
																		wrong += 1
						else:
												perfect += 1

						if total % 10000 == 0:
												print(F"""({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"""		)

						if skip_assert:
												return

						assert (
						    slow_ids == fast_ids
						), F"""line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(UpperCAmelCase_		)}\n{fast.tokenize(UpperCAmelCase_		)}"""

def 					_lowerCAmelCase (  lowercase_						,      lowercase_		):
						global batch_total
						for i in range(len(UpperCAmelCase_		)		):
												# premise, all languages
												for text in dataset[i]["premise"].values():
																		test_string(UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_		)

												# hypothesis, all languages
												for text in dataset[i]["hypothesis"]["translation"]:
																		test_string(UpperCAmelCase_						,      UpperCAmelCase_						,      UpperCAmelCase_		)


if __name__ == "__main__":
				for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
								snake_case_								=			list(slow_class.max_model_input_sizes.keys())
								for checkpoint in checkpoint_names:
												snake_case_								=			0
												snake_case_								=			0
												snake_case_								=			0
												snake_case_								=			0

												print(f'''========================== Checking {name}: {checkpoint} ==========================''')
												snake_case_								=			slow_class.from_pretrained(checkpoint, force_download=True)
												snake_case_								=			fast_class.from_pretrained(checkpoint, force_download=True)
												test_tokenizer(slow, fast)
												print(f'''Accuracy {perfect * 100 / total:.2f}''')










'''simple docstring'''







from __future__ import annotations

import unittest

from transformers import AlbertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
		from transformers.models.albert.modeling_tf_albert import (
		    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFAlbertForMaskedLM,
		    TFAlbertForMultipleChoice,
		    TFAlbertForPreTraining,
		    TFAlbertForQuestionAnswering,
		    TFAlbertForSequenceClassification,
		    TFAlbertForTokenClassification,
		    TFAlbertModel,
		)

class      lowerCamelCase__     :






	'''simple docstring'''

	def __init__(  self		:  List[str]   ,		__A		:  Dict   ,		__A		:  Optional[Any]=13   ,		__A		:  Dict=7   ,		__A		:  Dict=True   ,		__A		:  Any=True   ,		__A		:  Any=True   ,		__A		:  List[str]=True   ,		__A		:  Union[str, Any]=99   ,		__A		:  Optional[int]=16   ,		__A		:  Tuple=32   ,		__A		:  Optional[int]=2   ,		__A		:  Dict=4   ,		__A		:  Dict=37   ,		__A		:  Any="gelu"   ,		__A		:  Union[str, Any]=0.1   ,		__A		:  List[str]=0.1   ,		__A		:  Any=512   ,		__A		:  Optional[Any]=16   ,		__A		:  List[str]=2   ,		__A		:  List[str]=0.0_2   ,		__A		:  List[str]=3   ,		__A		:  Dict=4   ,		__A		:  List[str]=None   ,		)     ->			int:




								'''simple docstring'''





								lowerCAmelCase__  =   parent
								lowerCAmelCase__  =   13
								lowerCAmelCase__  =   7
								lowerCAmelCase__  =   True
								lowerCAmelCase__  =   True
								lowerCAmelCase__  =   True
								lowerCAmelCase__  =   True
								lowerCAmelCase__  =   99
								lowerCAmelCase__  =   16
								lowerCAmelCase__  =   32
								lowerCAmelCase__  =   2
								lowerCAmelCase__  =   4
								lowerCAmelCase__  =   37
								lowerCAmelCase__  =   """gelu"""
								lowerCAmelCase__  =   0.1
								lowerCAmelCase__  =   0.1
								lowerCAmelCase__  =   512
								lowerCAmelCase__  =   16
								lowerCAmelCase__  =   2
								lowerCAmelCase__  =   0.0_2
								lowerCAmelCase__  =   3
								lowerCAmelCase__  =   4
								lowerCAmelCase__  =   None




	def      lowercase__							(  self		:  Optional[int]					)     ->			int:




								'''simple docstring'''





								lowerCAmelCase__  =   ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size					)

								lowerCAmelCase__  =   None
								if self.use_input_mask:
															lowerCAmelCase__  =   random_attention_mask([self.batch_size, self.seq_length]					)

								lowerCAmelCase__  =   None
								if self.use_token_type_ids:
															lowerCAmelCase__  =   ids_tensor([self.batch_size, self.seq_length]   ,		self.type_vocab_size					)

								lowerCAmelCase__  =   None
								lowerCAmelCase__  =   None
								lowerCAmelCase__  =   None
								if self.use_labels:
															lowerCAmelCase__  =   ids_tensor([self.batch_size]   ,		self.type_sequence_label_size					)
															lowerCAmelCase__  =   ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels					)
															lowerCAmelCase__  =   ids_tensor([self.batch_size]   ,		self.num_choices					)

								lowerCAmelCase__  =   AlbertConfig(
								    vocab_size=self.vocab_size   ,		hidden_size=self.hidden_size   ,		embedding_size=self.embedding_size   ,		num_hidden_layers=self.num_hidden_layers   ,		num_attention_heads=self.num_attention_heads   ,		intermediate_size=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		hidden_dropout_prob=self.hidden_dropout_prob   ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,		max_position_embeddings=self.max_position_embeddings   ,		type_vocab_size=self.type_vocab_size   ,		initializer_range=self.initializer_range   ,		)

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




	def      lowercase__							(  self		:  Optional[Any]   ,		__A		:  Optional[Any]   ,		__A		:  Tuple   ,		__A		:  List[Any]   ,		__A		:  Tuple   ,		__A		:  Dict   ,		__A		:  Optional[int]   ,		__A		:  List[Any]					)     ->			Optional[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   TFAlbertModel(config=__A					)
								# inputs = {'input_ids': input_ids,
								#           'attention_mask': input_mask,
								#           'token_type_ids': token_type_ids}
								# sequence_output, pooled_output = model(**inputs)
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								lowerCAmelCase__  =   model(__A					)

								lowerCAmelCase__  =   [input_ids, input_mask]
								lowerCAmelCase__  =   model(__A					)

								lowerCAmelCase__  =   model(__A					)

								self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size)					)
								self.parent.assertEqual(result.pooler_output.shape   ,		(self.batch_size, self.hidden_size)					)




	def      lowercase__							(  self		:  List[str]   ,		__A		:  Optional[int]   ,		__A		:  Any   ,		__A		:  List[str]   ,		__A		:  Optional[int]   ,		__A		:  List[Any]   ,		__A		:  Optional[int]   ,		__A		:  Union[str, Any]					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   self.num_labels
								lowerCAmelCase__  =   TFAlbertForPreTraining(config=__A					)
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertEqual(result.prediction_logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size)					)
								self.parent.assertEqual(result.sop_logits.shape   ,		(self.batch_size, self.num_labels)					)




	def      lowercase__							(  self		:  str   ,		__A		:  Dict   ,		__A		:  Any   ,		__A		:  List[Any]   ,		__A		:  Tuple   ,		__A		:  Union[str, Any]   ,		__A		:  int   ,		__A		:  str					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   TFAlbertForMaskedLM(config=__A					)
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size)					)




	def      lowercase__							(  self		:  Dict   ,		__A		:  List[Any]   ,		__A		:  Union[str, Any]   ,		__A		:  Union[str, Any]   ,		__A		:  Optional[Any]   ,		__A		:  Optional[Any]   ,		__A		:  str   ,		__A		:  List[str]					)     ->			List[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.num_labels
								lowerCAmelCase__  =   TFAlbertForSequenceClassification(config=__A					)
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_labels)					)




	def      lowercase__							(  self		:  Dict   ,		__A		:  Any   ,		__A		:  Tuple   ,		__A		:  Union[str, Any]   ,		__A		:  List[Any]   ,		__A		:  List[Any]   ,		__A		:  Any   ,		__A		:  Any					)     ->			Dict:




								'''simple docstring'''





								lowerCAmelCase__  =   TFAlbertForQuestionAnswering(config=__A					)
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertEqual(result.start_logits.shape   ,		(self.batch_size, self.seq_length)					)
								self.parent.assertEqual(result.end_logits.shape   ,		(self.batch_size, self.seq_length)					)




	def      lowercase__							(  self		:  List[Any]   ,		__A		:  Union[str, Any]   ,		__A		:  List[Any]   ,		__A		:  Tuple   ,		__A		:  Dict   ,		__A		:  Tuple   ,		__A		:  List[str]   ,		__A		:  Dict					)     ->			List[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.num_choices
								lowerCAmelCase__  =   TFAlbertForMultipleChoice(config=__A					)
								lowerCAmelCase__  =   tf.tile(tf.expand_dims(__A   ,		1					)   ,		(1, self.num_choices, 1)					)
								lowerCAmelCase__  =   tf.tile(tf.expand_dims(__A   ,		1					)   ,		(1, self.num_choices, 1)					)
								lowerCAmelCase__  =   tf.tile(tf.expand_dims(__A   ,		1					)   ,		(1, self.num_choices, 1)					)
								lowerCAmelCase__  =   {
								    """input_ids""": multiple_choice_inputs_ids,
								    """attention_mask""": multiple_choice_input_mask,
								    """token_type_ids""": multiple_choice_token_type_ids,
								}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertListEqual(list(result["""logits"""].shape					)   ,		[self.batch_size, self.num_choices]					)




	def      lowercase__							(  self		:  int   ,		__A		:  Tuple   ,		__A		:  Optional[Any]   ,		__A		:  Any   ,		__A		:  Union[str, Any]   ,		__A		:  Tuple   ,		__A		:  str   ,		__A		:  Optional[Any]					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   self.num_labels
								lowerCAmelCase__  =   TFAlbertForTokenClassification(config=__A					)
								lowerCAmelCase__  =   {
								    """input_ids""": input_ids,
								    """attention_mask""": input_mask,
								    """token_type_ids""": token_type_ids,
								}
								lowerCAmelCase__  =   model(__A					)
								self.parent.assertListEqual(list(result["""logits"""].shape					)   ,		[self.batch_size, self.seq_length, self.num_labels]					)




	def      lowercase__							(  self		:  List[Any]					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   self.prepare_config_and_inputs()
								(
								    (
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,(
								    lowerCAmelCase__
								)  ,
								)  =   config_and_inputs
								lowerCAmelCase__  =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
								return config, inputs_dict





@require_tf
class      lowerCamelCase__     (     _A,     _A,     unittest.TestCase				):






	'''simple docstring'''

	A__		=	(
	    (
	        TFAlbertModel,
	        TFAlbertForPreTraining,
	        TFAlbertForMaskedLM,
	        TFAlbertForSequenceClassification,
	        TFAlbertForQuestionAnswering,
	        TFAlbertForTokenClassification,
	        TFAlbertForMultipleChoice,
	    )
	    if is_tf_available()
	    else ()
	)
	A__		=	(
	    {
	        '''feature-extraction''': TFAlbertModel,
	        '''fill-mask''': TFAlbertForMaskedLM,
	        '''question-answering''': TFAlbertForQuestionAnswering,
	        '''text-classification''': TFAlbertForSequenceClassification,
	        '''token-classification''': TFAlbertForTokenClassification,
	        '''zero-shot''': TFAlbertForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	A__		=	False
	A__		=	False




	def      lowercase__							(  self		:  Any   ,		__A		:  List[str]   ,		__A		:  int   ,		__A		:  Union[str, Any]=False					)     ->			Optional[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   super()._prepare_for_class(__A   ,		__A   ,		return_labels=__A					)

								if return_labels:
															if model_class in get_values(__A					):
																						lowerCAmelCase__  =   tf.zeros(self.model_tester.batch_size   ,		dtype=tf.intaa					)

								return inputs_dict




	def      lowercase__							(  self		:  Union[str, Any]					)     ->			List[str]:




								'''simple docstring'''





								lowerCAmelCase__  =   TFAlbertModelTester(self					)
								lowerCAmelCase__  =   ConfigTester(self   ,		config_class=__A   ,		hidden_size=37					)




	def      lowercase__							(  self		:  List[str]					)     ->			str:




								'''simple docstring'''





								self.config_tester.run_common_tests()




	def      lowercase__							(  self		:  Any					)     ->			List[str]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_model(*__A					)




	def      lowercase__							(  self		:  Optional[int]					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_for_pretraining(*__A					)




	def      lowercase__							(  self		:  List[Any]					)     ->			Dict:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_for_masked_lm(*__A					)




	def      lowercase__							(  self		:  Optional[int]					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_for_multiple_choice(*__A					)




	def      lowercase__							(  self		:  str					)     ->			List[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_for_sequence_classification(*__A					)




	def      lowercase__							(  self		:  Any					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_albert_for_question_answering(*__A					)




	@slow
	def      lowercase__							(  self		:  Tuple					)     ->			Dict:




								'''simple docstring'''





								for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															lowerCAmelCase__  =   TFAlbertModel.from_pretrained(__A					)
															self.assertIsNotNone(__A					)





@require_tf
class      lowerCamelCase__     (     unittest.TestCase				):






	'''simple docstring'''

	@slow
	def      lowercase__							(  self		:  Optional[int]					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   TFAlbertForPreTraining.from_pretrained("""albert-base-v2"""					)
								lowerCAmelCase__  =   tf.constant([[0, 1, 2, 3, 4, 5]]					)
								lowerCAmelCase__  =   model(__A					)[0]

								lowerCAmelCase__  =   [1, 6, 3_0000]
								self.assertEqual(output.shape   ,		__A					)

								lowerCAmelCase__  =   tf.constant(
								    [
								        [
								            [4.5_9_5_6_6_8, 0.7_4_4_6_2_7_5_4, -1.8_1_8_1_4_7],
								            [4.5_9_5_4_3_4_7, 0.7_4_5_4_1_8_4, -1.8_1_8_8_2_5_8],
								            [4.5_9_5_4_9_0_5, 0.7_4_4_8_2_3_5, -1.8_1_8_2_3_1_6],
								        ]
								    ]					)
								tf.debugging.assert_near(output[:, :3, :3]   ,		__A   ,		atol=1E-4					)





