


from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

UpperCAmelCase     =       {
    name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
}

UpperCAmelCase     =       datasets.load_dataset("""xnli""", split="""test+validation""")

UpperCAmelCase     =       0
UpperCAmelCase     =       0
UpperCAmelCase     =       0
UpperCAmelCase     =       0






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )->    Dict:




      """simple docstring"""

      if spm_diff == list(reversed(SCREAMING_SNAKE_CASE   )   ):
            # AAA -> AA+A vs A+AA case.
            return True
      elif len(SCREAMING_SNAKE_CASE   ) == len(SCREAMING_SNAKE_CASE   ) and fast.decode(SCREAMING_SNAKE_CASE   ) == fast.decode(SCREAMING_SNAKE_CASE   ):
            # Second order OK
            # Barrich -> Barr + ich vs Bar + rich
            return True
      snake_case_			= slow.encode(slow.decode(SCREAMING_SNAKE_CASE   )   )
      snake_case_			= fast.encode(fast.decode(SCREAMING_SNAKE_CASE   )   )
      if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
            # Type 3 error.
            # Snehagatha ->
            #       Sne, h, aga, th, a
            #       Sne, ha, gat, ha
            # Encoding the wrong with sp does not even recover what spm gave us
            # It fits tokenizer however...
            return True
      return False






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )->    List[Any]:




      """simple docstring"""

      snake_case_			= fast.encode_plus(SCREAMING_SNAKE_CASE   )[0]
      snake_case_			= enc.offsets
      snake_case_			,       snake_case_			= offsets[idx], offsets[idx - 1]
      if curr is not None and line[curr[0] : curr[1]] == "\u200f":
            return True
      if prev is not None and line[prev[0] : prev[1]] == "\u200f":
            return True






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )->    Any:




      """simple docstring"""

      for i, (spm_id, tok_id) in enumerate(zip(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )   ):
            if spm_id != tok_id:
                  break
      snake_case_			= i
      for i, (spm_id, tok_id) in enumerate(zip(reversed(SCREAMING_SNAKE_CASE   )   ,   reversed(SCREAMING_SNAKE_CASE   )   )   ):
            if spm_id != tok_id:
                  break
      snake_case_			= len(SCREAMING_SNAKE_CASE   ) - i

      snake_case_			= spm_ids[first:last]
      snake_case_			= tok_ids[first:last]

      if check_diff(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ):
            return True

      if check_LTR_mark(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ):
            return True

      if last - first > 5:
            # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
            snake_case_			= Counter(spm_ids[first:last]   )
            snake_case_			= Counter(tok_ids[first:last]   )

            snake_case_			= {spm_ for (spm_, si) in spms.items() if toks.get(spm_   ,   0   ) == si}
            snake_case_			= 3
            for i in range(last - first - min_width   ):
                  if all(spm_ids[first + i + j] in removable_tokens for j in range(SCREAMING_SNAKE_CASE   )   ):
                        snake_case_			= [
                            k
                            for k in range(last - first - min_width   )
                            if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
                        ]
                        for j in possible_matches:
                              if check_diff(spm_ids[first : first + i]   ,   tok_ids[first : first + j]   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ) and check_details(
                                  SCREAMING_SNAKE_CASE   ,   spm_ids[first + i : last]   ,   tok_ids[first + j : last]   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   ):
                                    return True

      print(f'''Spm: {[fast.decode([spm_ids[i]]   ) for i in range(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )]}'''   )
      try:
            print(f'''Tok: {[fast.decode([tok_ids[i]]   ) for i in range(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )]}'''   )
      except Exception:
            pass

      snake_case_			= fast.decode(spm_ids[:first]   )
      snake_case_			= fast.decode(spm_ids[last:]   )
      snake_case_			= fast.decode(spm_ids[first:last]   )
      print()
      print(SCREAMING_SNAKE_CASE   )
      return False






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )->    List[Any]:




      """simple docstring"""

      global perfect
      global imperfect
      global wrong
      global total

      snake_case_			= slow.encode(SCREAMING_SNAKE_CASE   )
      snake_case_			= fast.encode(SCREAMING_SNAKE_CASE   )

      snake_case_			= False
      total += 1

      if slow_ids != fast_ids:
            if check_details(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ):
                  snake_case_			= True
                  imperfect += 1
            else:
                  wrong += 1
      else:
            perfect += 1

      if total % 1_0000 == 0:
            print(f'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'''   )

      if skip_assert:
            return

      assert (
          slow_ids == fast_ids
      ), f'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(SCREAMING_SNAKE_CASE   )}\n{fast.tokenize(SCREAMING_SNAKE_CASE   )}'''






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )->    List[str]:




      """simple docstring"""

      global batch_total
      for i in range(len(SCREAMING_SNAKE_CASE   )   ):
            # premise, all languages
            for text in dataset[i]["premise"].values():
                  test_string(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )

            # hypothesis, all languages
            for text in dataset[i]["hypothesis"]["translation"]:
                  test_string(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
   for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
      UpperCAmelCase     =       list(slow_class.max_model_input_sizes.keys())
      for checkpoint in checkpoint_names:
         UpperCAmelCase     =       0
         UpperCAmelCase     =       0
         UpperCAmelCase     =       0
         UpperCAmelCase     =       0

         print(f'''========================== Checking {name}: {checkpoint} ==========================''')
         UpperCAmelCase     =       slow_class.from_pretrained(checkpoint, force_download=True)
         UpperCAmelCase     =       fast_class.from_pretrained(checkpoint, force_download=True)
         test_tokenizer(slow, fast)
         print(f'''Accuracy {perfect * 100 / total:.2f}''')



from collections import OrderedDict
from dataclasses import fields
from typing import Any, Tuple

import numpy as np

from .import_utils import is_torch_available






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   )->    Union[str, Any]:




      """simple docstring"""

      if is_torch_available():
            import torch

            if isinstance(SCREAMING_SNAKE_CASE   ,   torch.Tensor   ):
                  return True

      return isinstance(SCREAMING_SNAKE_CASE   ,   np.ndarray   )







class lowerCAmelCase_		(					lowerCamelCase__			):

 '''simple docstring'''

 def 		UpperCamelCase__	(      self   ):
       snake_case_			= fields(self   )

       # Safety and consistency checks
       if not len(_UpperCAmelCase   ):
             raise ValueError(F'''{self.__class__.__name__} has no fields.'''   )

       snake_case_			= getattr(self						, class_fields[0].name   )
       snake_case_			= all(getattr(self						, field.name   ) is None for field in class_fields[1:]   )

       if other_fields_are_none and isinstance(_UpperCAmelCase						, _UpperCAmelCase   ):
             for key, value in first_field.items():
                   snake_case_			= value
       else:
             for field in class_fields:
                   snake_case_			= getattr(self						, field.name   )
                   if v is not None:
                         snake_case_			= v



 def __delitem__(      self						, *_UpperCAmelCase						, **_UpperCAmelCase   ):
       raise Exception(F'''You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.'''   )



 def 		UpperCamelCase__	(      self						, *_UpperCAmelCase						, **_UpperCAmelCase   ):
       raise Exception(F'''You cannot use ``setdefault`` on a {self.__class__.__name__} instance.'''   )



 def 		UpperCamelCase__	(      self						, *_UpperCAmelCase						, **_UpperCAmelCase   ):
       raise Exception(F'''You cannot use ``pop`` on a {self.__class__.__name__} instance.'''   )



 def 		UpperCamelCase__	(      self						, *_UpperCAmelCase						, **_UpperCAmelCase   ):
       raise Exception(F'''You cannot use ``update`` on a {self.__class__.__name__} instance.'''   )



 def __getitem__(      self						, _UpperCAmelCase   ):
       if isinstance(_UpperCAmelCase						, _UpperCAmelCase   ):
             snake_case_			= dict(self.items()   )
             return inner_dict[k]
       else:
             return self.to_tuple()[k]



 def __setattr__(      self						, _UpperCAmelCase						, _UpperCAmelCase   ):
       if name in self.keys() and value is not None:
             # Don't call self.__setitem__ to avoid recursion errors
             super().__setitem__(_UpperCAmelCase						, _UpperCAmelCase   )
       super().__setattr__(_UpperCAmelCase						, _UpperCAmelCase   )



 def __setitem__(      self						, _UpperCAmelCase						, _UpperCAmelCase   ):
       # Will raise a KeyException if needed
       super().__setitem__(_UpperCAmelCase						, _UpperCAmelCase   )
       # Don't call self.__setattr__ to avoid recursion errors
       super().__setattr__(_UpperCAmelCase						, _UpperCAmelCase   )





 def 		UpperCamelCase__	(      self   ):
       return tuple(self[k] for k in self.keys()   )