



'''simple docstring'''







from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

_UpperCamelCase  						=			{
    name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
}

_UpperCamelCase  						=			datasets.load_dataset("""xnli""", split="""test+validation""")

_UpperCamelCase  						=			0
_UpperCamelCase  						=			0
_UpperCamelCase  						=			0
_UpperCamelCase  						=			0
def 			_lowerCAmelCase(      UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Optional[int]					, UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	Union[str, Any]						) ->							Optional[int]:
							if spm_diff == list(reversed(UpperCAmelCase_						)						):
														# AAA -> AA+A vs A+AA case.
														return True
							elif len(UpperCAmelCase_						) == len(UpperCAmelCase_						) and fast.decode(UpperCAmelCase_						) == fast.decode(UpperCAmelCase_						):
														# Second order OK
														# Barrich -> Barr + ich vs Bar + rich
														return True
							lowerCAmelCase__  =   slow.encode(slow.decode(UpperCAmelCase_						)						)
							lowerCAmelCase__  =   fast.encode(fast.decode(UpperCAmelCase_						)						)
							if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
														# Type 3 error.
														# Snehagatha ->
														#       Sne, h, aga, th, a
														#       Sne, ha, gat, ha
														# Encoding the wrong with sp does not even recover what spm gave us
														# It fits tokenizer however...
														return True
							return False
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Any					, UpperCAmelCase_      :	Optional[int]					, UpperCAmelCase_      :	str						) ->							str:
							lowerCAmelCase__  =   fast.encode_plus(UpperCAmelCase_						)[0]
							lowerCAmelCase__  =   enc.offsets
							lowerCAmelCase__  ,lowerCAmelCase__  =   offsets[idx], offsets[idx - 1]
							if curr is not None and line[curr[0] : curr[1]] == "\u200f":
														return True
							if prev is not None and line[prev[0] : prev[1]] == "\u200f":
														return True
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]					, UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	Optional[Any]						) ->							Tuple:
							# Encoding can be the same with same result AAA -> A + AA vs AA + A
							# We can check that we use at least exactly the same number of tokens.
							for i, (spm_id, tok_id) in enumerate(zip(UpperCAmelCase_					, UpperCAmelCase_						)						):
														if spm_id != tok_id:
																					break
							lowerCAmelCase__  =   i
							for i, (spm_id, tok_id) in enumerate(zip(reversed(UpperCAmelCase_						)					, reversed(UpperCAmelCase_						)						)						):
														if spm_id != tok_id:
																					break
							lowerCAmelCase__  =   len(UpperCAmelCase_						) - i

							lowerCAmelCase__  =   spm_ids[first:last]
							lowerCAmelCase__  =   tok_ids[first:last]

							if check_diff(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						):
														return True

							if check_LTR_mark(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						):
														return True

							if last - first > 5:
														# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
														lowerCAmelCase__  =   Counter(spm_ids[first:last]						)
														lowerCAmelCase__  =   Counter(tok_ids[first:last]						)

														lowerCAmelCase__  =   {spm_ for (spm_, si) in spms.items() if toks.get(spm_					, 0						) == si}
														lowerCAmelCase__  =   3
														for i in range(last - first - min_width						):
																					if all(spm_ids[first + i + j] in removable_tokens for j in range(UpperCAmelCase_						)						):
																												lowerCAmelCase__  =   [
																												    k
																												    for k in range(last - first - min_width						)
																												    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																												]
																												for j in possible_matches:
																																			if check_diff(spm_ids[first : first + i]					, tok_ids[first : first + j]					, UpperCAmelCase_					, UpperCAmelCase_						) and check_details(
																																			    UpperCAmelCase_					, spm_ids[first + i : last]					, tok_ids[first + j : last]					, UpperCAmelCase_					, UpperCAmelCase_					, ):
																																										return True

							print(F'''Spm: {[fast.decode([spm_ids[i]]						) for i in range(UpperCAmelCase_					, UpperCAmelCase_						)]}'''						)
							try:
														print(F'''Tok: {[fast.decode([tok_ids[i]]						) for i in range(UpperCAmelCase_					, UpperCAmelCase_						)]}'''						)
							except Exception:
														pass

							lowerCAmelCase__  =   fast.decode(spm_ids[:first]						)
							lowerCAmelCase__  =   fast.decode(spm_ids[last:]						)
							lowerCAmelCase__  =   fast.decode(spm_ids[first:last]						)
							print()
							print(UpperCAmelCase_						)
							return False
def 			_lowerCAmelCase(      UpperCAmelCase_      :	List[Any]					, UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Any						) ->							str:
							global perfect
							global imperfect
							global wrong
							global total

							lowerCAmelCase__  =   slow.encode(UpperCAmelCase_						)
							lowerCAmelCase__  =   fast.encode(UpperCAmelCase_						)

							lowerCAmelCase__  =   False
							total += 1

							if slow_ids != fast_ids:
														if check_details(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						):
																					lowerCAmelCase__  =   True
																					imperfect += 1
														else:
																					wrong += 1
							else:
														perfect += 1

							if total % 10000 == 0:
														print(F'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'''						)

							if skip_assert:
														return

							assert (
							    slow_ids == fast_ids
							), F'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(UpperCAmelCase_						)}\n{fast.tokenize(UpperCAmelCase_						)}'''
def 			_lowerCAmelCase(      UpperCAmelCase_      :	List[str]					, UpperCAmelCase_      :	Tuple						) ->							List[str]:
							global batch_total
							for i in range(len(UpperCAmelCase_						)						):
														# premise, all languages
														for text in dataset[i]["premise"].values():
																					test_string(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						)

														# hypothesis, all languages
														for text in dataset[i]["hypothesis"]["translation"]:
																					test_string(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_						)


if __name__ == "__main__":
		for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
				_UpperCamelCase  						=			list(slow_class.max_model_input_sizes.keys())
				for checkpoint in checkpoint_names:
						_UpperCamelCase  						=			0
						_UpperCamelCase  						=			0
						_UpperCamelCase  						=			0
						_UpperCamelCase  						=			0

						print(f'========================== Checking {name}: {checkpoint} ==========================')
						_UpperCamelCase  						=			slow_class.from_pretrained(checkpoint, force_download=True)
						_UpperCamelCase  						=			fast_class.from_pretrained(checkpoint, force_download=True)
						test_tokenizer(slow, fast)
						print(f'Accuracy {perfect * 100 / total:.2f}')










'''simple docstring'''







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
		import torch

if is_vision_available():
		from PIL import Image

		from transformers import DeiTImageProcessor

class      lowerCamelCase__     (     unittest.TestCase				):






	'''simple docstring'''

	def __init__(  self		:  List[str]   ,		__A		:  Any   ,		__A		:  Tuple=7   ,		__A		:  Dict=3   ,		__A		:  List[str]=18   ,		__A		:  Optional[int]=30   ,		__A		:  int=400   ,		__A		:  Union[str, Any]=True   ,		__A		:  Tuple=None   ,		__A		:  int=True   ,		__A		:  int=None   ,		__A		:  List[str]=True   ,		__A		:  Union[str, Any]=[0.5, 0.5, 0.5]   ,		__A		:  List[Any]=[0.5, 0.5, 0.5]   ,		)     ->			List[str]:




								'''simple docstring'''





								lowerCAmelCase__  =   size if size is not None else {"""height""": 20, """width""": 20}
								lowerCAmelCase__  =   crop_size if crop_size is not None else {"""height""": 18, """width""": 18}

								lowerCAmelCase__  =   parent
								lowerCAmelCase__  =   batch_size
								lowerCAmelCase__  =   num_channels
								lowerCAmelCase__  =   image_size
								lowerCAmelCase__  =   min_resolution
								lowerCAmelCase__  =   max_resolution
								lowerCAmelCase__  =   do_resize
								lowerCAmelCase__  =   size
								lowerCAmelCase__  =   do_center_crop
								lowerCAmelCase__  =   crop_size
								lowerCAmelCase__  =   do_normalize
								lowerCAmelCase__  =   image_mean
								lowerCAmelCase__  =   image_std




	def      lowercase__							(  self		:  Tuple					)     ->			Optional[int]:




								'''simple docstring'''





								return {
								    "do_resize": self.do_resize,
								    "size": self.size,
								    "do_center_crop": self.do_center_crop,
								    "crop_size": self.crop_size,
								    "do_normalize": self.do_normalize,
								    "image_mean": self.image_mean,
								    "image_std": self.image_std,
								}





@require_torch
@require_vision
class      lowerCamelCase__     (     _A,     unittest.TestCase				):






	'''simple docstring'''

	A__		=	DeiTImageProcessor if is_vision_available() else None
	A__		=	True




	def      lowercase__							(  self		:  List[Any]					)     ->			Dict:




								'''simple docstring'''





								lowerCAmelCase__  =   DeiTImageProcessingTester(self					)




	@property
	def      lowercase__							(  self		:  Any					)     ->			int:




								'''simple docstring'''





								return self.image_processor_tester.prepare_image_processor_dict()




	def      lowercase__							(  self		:  str					)     ->			Optional[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.image_processing_class(**self.image_processor_dict					)
								self.assertTrue(hasattr(__A   ,		"""do_resize"""					)					)
								self.assertTrue(hasattr(__A   ,		"""size"""					)					)
								self.assertTrue(hasattr(__A   ,		"""do_center_crop"""					)					)
								self.assertTrue(hasattr(__A   ,		"""center_crop"""					)					)
								self.assertTrue(hasattr(__A   ,		"""do_normalize"""					)					)
								self.assertTrue(hasattr(__A   ,		"""image_mean"""					)					)
								self.assertTrue(hasattr(__A   ,		"""image_std"""					)					)




	def      lowercase__							(  self		:  Optional[int]					)     ->			Optional[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.image_processing_class.from_dict(self.image_processor_dict					)
								self.assertEqual(image_processor.size   ,		{"""height""": 20, """width""": 20}					)
								self.assertEqual(image_processor.crop_size   ,		{"""height""": 18, """width""": 18}					)

								lowerCAmelCase__  =   self.image_processing_class.from_dict(self.image_processor_dict   ,		size=42   ,		crop_size=84					)
								self.assertEqual(image_processor.size   ,		{"""height""": 42, """width""": 42}					)
								self.assertEqual(image_processor.crop_size   ,		{"""height""": 84, """width""": 84}					)




	def      lowercase__							(  self		:  Optional[int]					)     ->			List[str]:




								'''simple docstring'''





								pass




	def      lowercase__							(  self		:  Dict					)     ->			Tuple:




								'''simple docstring'''





								lowerCAmelCase__  =   self.image_processing_class(**self.image_processor_dict					)
								# create random PIL images
								lowerCAmelCase__  =   prepare_image_inputs(self.image_processor_tester   ,		equal_resolution=__A					)
								for image in image_inputs:
															self.assertIsInstance(__A   ,		Image.Image					)

								# Test not batched input
								lowerCAmelCase__  =   image_processing(image_inputs[0]   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)

								# Test batched
								lowerCAmelCase__  =   image_processing(__A   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)




	def      lowercase__							(  self		:  List[Any]					)     ->			Optional[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   self.image_processing_class(**self.image_processor_dict					)
								# create random numpy tensors
								lowerCAmelCase__  =   prepare_image_inputs(self.image_processor_tester   ,		equal_resolution=__A   ,		numpify=__A					)
								for image in image_inputs:
															self.assertIsInstance(__A   ,		np.ndarray					)

								# Test not batched input
								lowerCAmelCase__  =   image_processing(image_inputs[0]   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)

								# Test batched
								lowerCAmelCase__  =   image_processing(__A   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)




	def      lowercase__							(  self		:  List[str]					)     ->			Tuple:




								'''simple docstring'''





								lowerCAmelCase__  =   self.image_processing_class(**self.image_processor_dict					)
								# create random PyTorch tensors
								lowerCAmelCase__  =   prepare_image_inputs(self.image_processor_tester   ,		equal_resolution=__A   ,		torchify=__A					)
								for image in image_inputs:
															self.assertIsInstance(__A   ,		torch.Tensor					)

								# Test not batched input
								lowerCAmelCase__  =   image_processing(image_inputs[0]   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)

								# Test batched
								lowerCAmelCase__  =   image_processing(__A   ,		return_tensors="""pt"""					).pixel_values
								self.assertEqual(
								    encoded_images.shape   ,		(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size["""height"""],
								        self.image_processor_tester.crop_size["""width"""],
								    )   ,		)





