


from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

_snake_case				=			{
    name: (getattr(transformers, name), getattr(transformers, name + "Fast")) for name in SLOW_TO_FAST_CONVERTERS
}

_snake_case				=			datasets.load_dataset("xnli", split="test+validation")

_snake_case				=			0
_snake_case				=			0
_snake_case				=			0
_snake_case				=			0







def 	A (		A__		,		A__		,		A__		,		A__			):
     if spm_diff == list(reversed(A__			)			):
          # AAA -> AA+A vs A+AA case.
          return True
     elif len(A__			) == len(A__			) and fast.decode(A__			) == fast.decode(A__			):
          # Second order OK
          # Barrich -> Barr + ich vs Bar + rich
          return True
     UpperCAmelCase    :					List[Any]			=      slow.encode(slow.decode(A__			)			)
     UpperCAmelCase    :					str			=      fast.encode(fast.decode(A__			)			)
     if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
          # Type 3 error.
          # Snehagatha ->
          #       Sne, h, aga, th, a
          #       Sne, ha, gat, ha
          # Encoding the wrong with sp does not even recover what spm gave us
          # It fits tokenizer however...
          return True
     return False







def 	A (		A__		,		A__		,		A__			):
     UpperCAmelCase    :					Tuple			=      fast.encode_plus(A__			)[0]
     UpperCAmelCase    :					List[Any]			=      enc.offsets
     UpperCAmelCase	,			UpperCAmelCase    :					Any			=      offsets[idx], offsets[idx - 1]
     if curr is not None and line[curr[0] : curr[1]] == "\u200f":
          return True
     if prev is not None and line[prev[0] : prev[1]] == "\u200f":
          return True







def 	A (		A__		,		A__		,		A__		,		A__		,		A__			):
     # Encoding can be the same with same result AAA -> A + AA vs AA + A
     # We can check that we use at least exactly the same number of tokens.
     for i, (spm_id, tok_id) in enumerate(zip(A__		,		A__			)			):
          if spm_id != tok_id:
               break
     UpperCAmelCase    :					int			=      i
     for i, (spm_id, tok_id) in enumerate(zip(reversed(A__			)		,		reversed(A__			)			)			):
          if spm_id != tok_id:
               break
     UpperCAmelCase    :					List[str]			=      len(A__			) - i

     UpperCAmelCase    :					int			=      spm_ids[first:last]
     UpperCAmelCase    :					Optional[Any]			=      tok_ids[first:last]

     if check_diff(A__		,		A__		,		A__		,		A__			):
          return True

     if check_LTR_mark(A__		,		A__		,		A__			):
          return True

     if last - first > 5:
          # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
          UpperCAmelCase    :					List[Any]			=      Counter(spm_ids[first:last]			)
          UpperCAmelCase    :					str			=      Counter(tok_ids[first:last]			)

          UpperCAmelCase    :					Any			=      {spm_ for (spm_, si) in spms.items() if toks.get(spm_		,		0			) == si}
          UpperCAmelCase    :					int			=      3
          for i in range(last - first - min_width			):
               if all(spm_ids[first + i + j] in removable_tokens for j in range(A__			)			):
                    UpperCAmelCase    :					List[Any]			=      [
                        k
                        for k in range(last - first - min_width			)
                        if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
                    ]
                    for j in possible_matches:
                         if check_diff(spm_ids[first : first + i]		,		tok_ids[first : first + j]		,		A__		,		A__			) and check_details(
                             A__		,		spm_ids[first + i : last]		,		tok_ids[first + j : last]		,		A__		,		A__		,		):
                              return True

     print(f'Spm: {[fast.decode([spm_ids[i]]			) for i in range(A__		,		A__			)]}'			)
     try:
          print(f'Tok: {[fast.decode([tok_ids[i]]			) for i in range(A__		,		A__			)]}'			)
     except Exception:
          pass

     UpperCAmelCase    :					List[str]			=      fast.decode(spm_ids[:first]			)
     UpperCAmelCase    :					List[str]			=      fast.decode(spm_ids[last:]			)
     UpperCAmelCase    :					List[Any]			=      fast.decode(spm_ids[first:last]			)
     print()
     print(A__			)
     return False







def 	A (		A__		,		A__		,		A__			):
     global perfect
     global imperfect
     global wrong
     global total

     UpperCAmelCase    :					int			=      slow.encode(A__			)
     UpperCAmelCase    :					Union[str, Any]			=      fast.encode(A__			)

     UpperCAmelCase    :					List[Any]			=      False
     total += 1

     if slow_ids != fast_ids:
          if check_details(A__		,		A__		,		A__		,		A__		,		A__			):
               UpperCAmelCase    :					Optional[int]			=      True
               imperfect += 1
          else:
               wrong += 1
     else:
          perfect += 1

     if total % 1_0_0_0_0 == 0:
          print(f'({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'			)

     if skip_assert:
          return

     assert (
         slow_ids == fast_ids
     ), f'line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(A__			)}\n{fast.tokenize(A__			)}'







def 	A (		A__		,		A__			):
     global batch_total
     for i in range(len(A__			)			):
          # premise, all languages
          for text in dataset[i]["premise"].values():
               test_string(A__		,		A__		,		A__			)

          # hypothesis, all languages
          for text in dataset[i]["hypothesis"]["translation"]:
               test_string(A__		,		A__		,		A__			)


if __name__ == "__main__":
      for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
            _snake_case				=			list(slow_class.max_model_input_sizes.keys())
            for checkpoint in checkpoint_names:
                  _snake_case				=			0
                  _snake_case				=			0
                  _snake_case				=			0
                  _snake_case				=			0

                  print(f'''========================== Checking {name}: {checkpoint} ==========================''')
                  _snake_case				=			slow_class.from_pretrained(checkpoint, force_download=True)
                  _snake_case				=			fast_class.from_pretrained(checkpoint, force_download=True)
                  test_tokenizer(slow, fast)
                  print(f'''Accuracy {perfect * 1_0_0 / total:.2f}''')







import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand




class  A_ (			a_  ):







 """simple docstring"""





 @staticmethod
 def 	_UpperCamelCase     (      _a       :			ArgumentParser					)     ->      str:
      UpperCAmelCase    :					str			=      parser.add_parser("login"							,     help="Log in using the same credentials as on huggingface.co"					)
      login_parser.set_defaults(func=lambda _a					: LoginCommand(_a					)					)
      UpperCAmelCase    :					str			=      parser.add_parser("whoami"							,     help="Find out which huggingface.co account you are logged in as."					)
      whoami_parser.set_defaults(func=lambda _a					: WhoamiCommand(_a					)					)
      UpperCAmelCase    :					List[Any]			=      parser.add_parser("logout"							,     help="Log out"					)
      logout_parser.set_defaults(func=lambda _a					: LogoutCommand(_a					)					)

      # new system: git-based repo system
      UpperCAmelCase    :					List[str]			=      parser.add_parser(
          "repo"							,     help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos."							,     )
      UpperCAmelCase    :					List[str]			=      repo_parser.add_subparsers(
          help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands"					)
      UpperCAmelCase    :					Optional[Any]			=      repo_subparsers.add_parser(
          "create"							,     help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co"					)
      repo_create_parser.add_argument(
          "name"							,     type=_a							,     help="Name for your model's repo. Will be namespaced under your username to build the model id."							,     )
      repo_create_parser.add_argument("--organization"							,     type=_a							,     help="Optional: organization namespace."					)
      repo_create_parser.add_argument("-y"							,     "--yes"							,     action="store_true"							,     help="Optional: answer Yes to the prompt"					)
      repo_create_parser.set_defaults(func=lambda _a					: RepoCreateCommand(_a					)					)




class  A_ :







 """simple docstring"""





 _UpperCamelCase					:			str     	=       '''\u001b[1m'''
 _UpperCamelCase					:			Optional[Any]     	=       '''\u001b[31m'''
 _UpperCamelCase					:			List[Any]     	=       '''\u001b[90m'''
 _UpperCamelCase					:			Tuple     	=       '''\u001b[0m'''
 @classmethod
 def 	_UpperCamelCase     (      cls       :			Any							,     _a       :			Tuple					)     ->      Any:
      return f'{cls._bold}{s}{cls._reset}'
 @classmethod
 def 	_UpperCamelCase     (      cls       :			Optional[Any]							,     _a       :			List[Any]					)     ->      str:
      return f'{cls._bold}{cls._red}{s}{cls._reset}'

 @classmethod
 def 	_UpperCamelCase     (      cls       :			str							,     _a       :			Optional[Any]					)     ->      Tuple:
      return f'{cls._gray}{s}{cls._reset}'







def 	A (		A__		,		A__			):
     UpperCAmelCase    :					str			=      [max(len(str(A__			)			) for x in col			) for col in zip(*A__		,		A__			)]
     UpperCAmelCase    :					str			=      ("{{:{}}} " * len(A__			)).format(*A__			)
     UpperCAmelCase    :					Dict			=      []
     lines.append(row_format.format(*A__			)			)
     lines.append(row_format.format(*["-" * w for w in col_widths]			)			)
     for row in rows:
          lines.append(row_format.format(*A__			)			)
     return "\n".join(A__			)




class  A_ :







 """simple docstring"""





 def __init__(      self       :			List[str]							,     _a       :			str					)     ->      str:
      UpperCAmelCase    :					str			=      args




class  A_ (			a_  ):







 """simple docstring"""





 def 	_UpperCamelCase     (      self       :			Optional[int]					)     ->      Any:
      print(
          ANSI.red(
              "ERROR! `huggingface-cli login` uses an outdated login mechanism "
              "that is not compatible with the Hugging Face Hub backend anymore. "
              "Please use `huggingface-cli login instead."					)					)




class  A_ (			a_  ):







 """simple docstring"""





 def 	_UpperCamelCase     (      self       :			Tuple					)     ->      Tuple:
      print(
          ANSI.red(
              "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
              "`huggingface-cli whoami` instead."					)					)
      UpperCAmelCase    :					Dict			=      HfFolder.get_token()
      if token is None:
           print("Not logged in"					)
           exit()
      try:
           UpperCAmelCase	,			UpperCAmelCase    :					Optional[int]			=      whoami(_a					)
           print(_a					)
           if orgs:
                print(ANSI.bold("orgs: "					)							,     ",".join(_a					)					)
      except HTTPError as e:
           print(_a					)
           print(ANSI.red(e.response.text					)					)
           exit(1					)




class  A_ (			a_  ):







 """simple docstring"""





 def 	_UpperCamelCase     (      self       :			List[Any]					)     ->      int:
      print(
          ANSI.red(
              "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
              "that is not compatible with the Hugging Face Hub backend anymore. "
              "Please use `huggingface-cli logout instead."					)					)




class  A_ (			a_  ):







 """simple docstring"""





 def 	_UpperCamelCase     (      self       :			List[str]					)     ->      Union[str, Any]:
      print(
          ANSI.red(
              "WARNING! Managing repositories through transformers-cli is deprecated. "
              "Please use `huggingface-cli` instead."					)					)
      UpperCAmelCase    :					List[Any]			=      HfFolder.get_token()
      if token is None:
           print("Not logged in"					)
           exit(1					)
      try:
           UpperCAmelCase    :					List[str]			=      subprocess.check_output(["git", "--version"]					).decode("utf-8"					)
           print(ANSI.gray(stdout.strip()					)					)
      except FileNotFoundError:
           print("Looks like you do not have git installed, please install."					)

      try:
           UpperCAmelCase    :					Tuple			=      subprocess.check_output(["git-lfs", "--version"]					).decode("utf-8"					)
           print(ANSI.gray(stdout.strip()					)					)
      except FileNotFoundError:
           print(
               ANSI.red(
                   "Looks like you do not have git-lfs installed, please install."
                   " You can install from https://git-lfs.github.com/."
                   " Then run `git lfs install` (you only have to do this once)."					)					)
      print(""					)

      UpperCAmelCase	,			UpperCAmelCase    :					Tuple			=      whoami(_a					)
      UpperCAmelCase    :					Union[str, Any]			=      self.args.organization if self.args.organization is not None else user
      UpperCAmelCase    :					Dict			=      f'{namespace}/{self.args.name}'
      print(f'You are about to create {ANSI.bold(_a					)}'					)

      if not self.args.yes:
           UpperCAmelCase    :					Any			=      input("Proceed? [Y/n] "					).lower()
           if not (choice == "" or choice == "y" or choice == "yes"):
                print("Abort"					)
                exit()
      try:
           UpperCAmelCase    :					List[Any]			=      create_repo(_a							,     name=self.args.name							,     organization=self.args.organization					)
      except HTTPError as e:
           print(_a					)
           print(ANSI.red(e.response.text					)					)
           exit(1					)
      print("\nYour repo now lives at:"					)
      print(f'  {ANSI.bold(_a					)}'					)
      print("\nYou can clone it locally with the command below, and commit/push as usual."					)
      print(f'\n  git clone {url}'					)
      print(""					)



