





"""simple docstring"""


from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

a__       :  List[str]       		=					{
    name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
}

a__       :  Tuple       		=					datasets.load_dataset("""xnli""", split="""test+validation""")

a__       :  Union[str, Any]       		=					0
a__       :  List[str]       		=					0
a__       :  int       		=					0
a__       :  Optional[int]       		=					0





def 				a__      (	_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)     ->	Dict:

 '''simple docstring'''






 if spm_diff == list(reversed(_UpperCAmelCase						)						):
  # AAA -> AA+A vs A+AA case.
  return True
 elif len(_UpperCAmelCase						) == len(_UpperCAmelCase						) and fast.decode(_UpperCAmelCase						) == fast.decode(_UpperCAmelCase						):
  # Second order OK
  # Barrich -> Barr + ich vs Bar + rich
  return True
 snake_case__     				=					slow.encode(slow.decode(_UpperCAmelCase						)						)
 snake_case__     				=					fast.encode(fast.decode(_UpperCAmelCase						)						)
 if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
  # Type 3 error.
  # Snehagatha ->
  #       Sne, h, aga, th, a
  #       Sne, ha, gat, ha
  # Encoding the wrong with sp does not even recover what spm gave us
  # It fits tokenizer however...
  return True
 return False





def 				a__      (	_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)     ->	str:

 '''simple docstring'''






 snake_case__     				=					fast.encode_plus(_UpperCAmelCase						)[0]
 snake_case__     				=					enc.offsets
 snake_case__				, snake_case__     				=					offsets[idx], offsets[idx - 1]
 if curr is not None and line[curr[0] : curr[1]] == "\u200f":
  return True
 if prev is not None and line[prev[0] : prev[1]] == "\u200f":
  return True





def 				a__      (	_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)     ->	int:

 '''simple docstring'''






 for i, (spm_id, tok_id) in enumerate(zip(_UpperCAmelCase						,  _UpperCAmelCase						)						):
  if spm_id != tok_id:
   break
 snake_case__     				=					i
 for i, (spm_id, tok_id) in enumerate(zip(reversed(_UpperCAmelCase						)						,  reversed(_UpperCAmelCase						)						)						):
  if spm_id != tok_id:
   break
 snake_case__     				=					len(_UpperCAmelCase						) - i

 snake_case__     				=					spm_ids[first:last]
 snake_case__     				=					tok_ids[first:last]

 if check_diff(_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						):
  return True

 if check_LTR_mark(_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						):
  return True

 if last - first > 5:
  # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
  snake_case__     				=					Counter(spm_ids[first:last]						)
  snake_case__     				=					Counter(tok_ids[first:last]						)

  snake_case__     				=					{spm_ for (spm_, si) in spms.items() if toks.get(spm_						,  0						) == si}
  snake_case__     				=					3
  for i in range(last - first - min_width						):
   if all(spm_ids[first + i + j] in removable_tokens for j in range(_UpperCAmelCase						)						):
    snake_case__     				=					[
        k
        for k in range(last - first - min_width						)
        if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
    ]
    for j in possible_matches:
     if check_diff(spm_ids[first : first + i]						,  tok_ids[first : first + j]						,  _UpperCAmelCase						,  _UpperCAmelCase						) and check_details(
         _UpperCAmelCase						,  spm_ids[first + i : last]						,  tok_ids[first + j : last]						,  _UpperCAmelCase						,  _UpperCAmelCase						,  ):
      return True

 print(f"Spm: {[fast.decode([spm_ids[i]]						) for i in range(_UpperCAmelCase						,  _UpperCAmelCase						)]}"						)
 try:
  print(f"Tok: {[fast.decode([tok_ids[i]]						) for i in range(_UpperCAmelCase						,  _UpperCAmelCase						)]}"						)
 except Exception:
  pass

 snake_case__     				=					fast.decode(spm_ids[:first]						)
 snake_case__     				=					fast.decode(spm_ids[last:]						)
 snake_case__     				=					fast.decode(spm_ids[first:last]						)
 print()
 print(_UpperCAmelCase						)
 return False





def 				a__      (	_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)     ->	Union[str, Any]:

 '''simple docstring'''






 global perfect
 global imperfect
 global wrong
 global total

 snake_case__     				=					slow.encode(_UpperCAmelCase						)
 snake_case__     				=					fast.encode(_UpperCAmelCase						)

 snake_case__     				=					False
 total += 1

 if slow_ids != fast_ids:
  if check_details(_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						):
   snake_case__     				=					True
   imperfect += 1
  else:
   wrong += 1
 else:
  perfect += 1

 if total % 1_0_0_0_0 == 0:
  print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"						)

 if skip_assert:
  return

 assert (
     slow_ids == fast_ids
 ), f"line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(_UpperCAmelCase						)}\n{fast.tokenize(_UpperCAmelCase						)}"





def 				a__      (	_UpperCAmelCase						,  _UpperCAmelCase						)     ->	Tuple:

 '''simple docstring'''






 global batch_total
 for i in range(len(_UpperCAmelCase						)						):
  # premise, all languages
  for text in dataset[i]["premise"].values():
   test_string(_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)

  # hypothesis, all languages
  for text in dataset[i]["hypothesis"]["translation"]:
   test_string(_UpperCAmelCase						,  _UpperCAmelCase						,  _UpperCAmelCase						)


if __name__ == "__main__":
   for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
      a__       :  Dict       		=					list(slow_class.max_model_input_sizes.keys())
      for checkpoint in checkpoint_names:
         a__       :  int       		=					0
         a__       :  Tuple       		=					0
         a__       :  List[Any]       		=					0
         a__       :  Any       		=					0

         print(f'''========================== Checking {name}: {checkpoint} ==========================''')
         a__       :  Optional[int]       		=					slow_class.from_pretrained(checkpoint, force_download=True)
         a__       :  List[Any]       		=					fast_class.from_pretrained(checkpoint, force_download=True)
         test_tokenizer(slow, fast)
         print(f'''Accuracy {perfect * 1_0_0 / total:.2f}''')








"""simple docstring"""


def 				a__      (	_UpperCAmelCase						)     ->	int:

 '''simple docstring'''






 snake_case__     				=					0
 for plain_chr in plain_text:
  snake_case__     				=					(
      ord(_UpperCAmelCase						) + (hash_value << 6) + (hash_value << 1_6) - hash_value
  )
 return hash_value

