
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__magic_name__				( UpperCAmelCase_						,unittest.TestCase					):


  """simple docstring"""





  _UpperCamelCase						 = PreTrainedTokenizerFast
  _UpperCamelCase						 = False
  _UpperCamelCase						 = True
  _UpperCamelCase						 = "tokenizer_file"


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						False  # because we don't have pretrained_vocab_files_map
    super().setUp()
    _lowerCamelCase      =						True

    _lowerCamelCase      =						['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
    _lowerCamelCase      =						'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

    # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
    _lowerCamelCase      =						[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

    _lowerCamelCase      =						PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
    tokenizer.save_pretrained(self.tmpdirname		)


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
    # model
    pass


  def     _UpperCAmelCase  (					self		):
    # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
    pass


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.tmpdirname
    # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
        try:
          _lowerCamelCase      =						tempfile.mkdtemp()
          _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase	,		**__UpperCamelCase		)

          tokenizer.save_pretrained(self.tmpdirname		)
          super().test_training_new_tokenizer()
        finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname		)
          _lowerCamelCase      =						tmpdirname_orig


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.tmpdirname
    # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
        try:
          _lowerCamelCase      =						tempfile.mkdtemp()
          _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase	,		**__UpperCamelCase		)

          tokenizer.save_pretrained(self.tmpdirname		)
          super().test_training_new_tokenizer_with_special_tokens_change()
        finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname		)
          _lowerCamelCase      =						tmpdirname_orig


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)

    _lowerCamelCase      =						('''a''' for _ in range(10_00		))
    _lowerCamelCase      =						tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase	,		length=10_00	,		vocab_size=50		)

    _lowerCamelCase      =						new_tokenizer.encode('''a🤗'''		)
    self.assertEqual(__UpperCamelCase	,		[64, 1_72, 2_53, 97, 2_45]		)







@require_tokenizers
class 		__magic_name__				( unittest.TestCase					):


  """simple docstring"""





  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						AutoTokenizer.from_pretrained('''bert-base-cased'''		)
    _lowerCamelCase      =						json.loads(tokenizer._tokenizer.to_str()		)
    _lowerCamelCase      =						len(__UpperCamelCase		)

    with tempfile.TemporaryDirectory() as tmp_dir:
      # Hack to save this in the tokenizer_config.json
      _lowerCamelCase      =						['''tokenizer.4.0.0.json''']
      tokenizer.save_pretrained(__UpperCamelCase		)
      json.dump(__UpperCamelCase	,		open(os.path.join(__UpperCamelCase	,		'''tokenizer.4.0.0.json'''		)	,		'''w'''		)		)

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      _lowerCamelCase      =						AutoTokenizer.from_pretrained(__UpperCamelCase		)
      self.assertEqual(len(__UpperCamelCase		)	,		len(__UpperCamelCase		) + 1		)
      _lowerCamelCase      =						json.loads(new_tokenizer._tokenizer.to_str()		)
      self.assertIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)

      # Will need to be adjusted if we reach v42 and this test is still here.
      # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
      shutil.move(os.path.join(__UpperCamelCase	,		'''tokenizer.4.0.0.json'''		)	,		os.path.join(__UpperCamelCase	,		'''tokenizer.42.0.0.json'''		)		)
      _lowerCamelCase      =						['''tokenizer.42.0.0.json''']
      tokenizer.save_pretrained(__UpperCamelCase		)
      _lowerCamelCase      =						AutoTokenizer.from_pretrained(__UpperCamelCase		)
      self.assertEqual(len(__UpperCamelCase		)	,		len(__UpperCamelCase		)		)
      _lowerCamelCase      =						json.loads(new_tokenizer._tokenizer.to_str()		)
      self.assertNotIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)


  def     _UpperCAmelCase  (					self		):
    # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
    _lowerCamelCase      =						'''hf-internal-testing/test-two-tokenizers'''

    # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
    _lowerCamelCase      =						AutoTokenizer.from_pretrained(__UpperCamelCase		)
    self.assertEqual(len(__UpperCamelCase		)	,		2_89_97		)
    _lowerCamelCase      =						json.loads(tokenizer._tokenizer.to_str()		)
    self.assertIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)

    # Testing an older version by monkey-patching the version in the module it's used.
    import transformers as old_transformers

    _lowerCamelCase      =						'''3.0.0'''
    _lowerCamelCase      =						old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase		)
    self.assertEqual(len(__UpperCamelCase		)	,		2_89_96		)
    _lowerCamelCase      =						json.loads(old_tokenizer._tokenizer.to_str()		)
    self.assertNotIn('''huggingface'''	,		json_tokenizer['''model''']['''vocab''']		)







@require_tokenizers
class 		__magic_name__				( unittest.TestCase					):


  """simple docstring"""





  def     _UpperCAmelCase  (					self		):
    # See https://github.com/huggingface/transformers/pull/12550
    # and https://github.com/huggingface/tokenizers/issues/537
    _lowerCamelCase      =						PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''		)
    _lowerCamelCase      =						'''The Matrix is a 1999 science fiction action film.'''

    with concurrent.futures.ThreadPoolExecutor() as executor:
      _lowerCamelCase      =						[executor.submit(self.fetch	,		__UpperCamelCase	,		__UpperCamelCase		) for i in range(10		)]
      _lowerCamelCase      =						[future.result() for future in futures]
      self.assertEqual(__UpperCamelCase	,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    return tokenizer.encode(__UpperCamelCase	,		truncation='''longest_first'''	,		padding='''longest'''		)










'''simple docstring'''







import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					@staticmethod
					def 							__SCREAMING_SNAKE_CASE      (       __UpperCamelCase					):
							_snake_case      			=			parser.add_parser("login",    help="Log in using the same credentials as on huggingface.co"					)
							login_parser.set_defaults(func=lambda __UpperCamelCase					: LoginCommand(__UpperCamelCase					)					)
							_snake_case      			=			parser.add_parser("whoami",    help="Find out which huggingface.co account you are logged in as."					)
							whoami_parser.set_defaults(func=lambda __UpperCamelCase					: WhoamiCommand(__UpperCamelCase					)					)
							_snake_case      			=			parser.add_parser("logout",    help="Log out"					)
							logout_parser.set_defaults(func=lambda __UpperCamelCase					: LogoutCommand(__UpperCamelCase					)					)

							# new system: git-based repo system
							_snake_case      			=			parser.add_parser(
							    "repo",    help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.",    )
							_snake_case      			=			repo_parser.add_subparsers(
							    help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands"					)
							_snake_case      			=			repo_subparsers.add_parser(
							    "create",    help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co"					)
							repo_create_parser.add_argument(
							    "name",    type=__UpperCamelCase,    help="Name for your model's repo. Will be namespaced under your username to build the model id.",    )
							repo_create_parser.add_argument("--organization",    type=__UpperCamelCase,    help="Optional: organization namespace."					)
							repo_create_parser.add_argument("-y",    "--yes",    action="store_true",    help="Optional: answer Yes to the prompt"					)
							repo_create_parser.set_defaults(func=lambda __UpperCamelCase					: RepoCreateCommand(__UpperCamelCase					)					)





class 	lowerCamelCase_				:




					'''simple docstring'''






					lowercase__       =					"""\u001b[1m"""
					lowercase__       =					"""\u001b[31m"""
					lowercase__       =					"""\u001b[90m"""
					lowercase__       =					"""\u001b[0m"""






					@classmethod
					def 							__SCREAMING_SNAKE_CASE      (       cls,    __UpperCamelCase					):
							return f'''{cls._bold}{s}{cls._reset}'''






					@classmethod
					def 							__SCREAMING_SNAKE_CASE      (       cls,    __UpperCamelCase					):
							return f'''{cls._bold}{cls._red}{s}{cls._reset}'''






					@classmethod
					def 							__SCREAMING_SNAKE_CASE      (       cls,    __UpperCamelCase					):
							return f'''{cls._gray}{s}{cls._reset}'''






def 						_lowercase						(      lowercase_		,				lowercase_    ):



		'''simple docstring'''







		_snake_case      			=			[max(len(str(lowercase_    )    ) for x in col    ) for col in zip(*lowercase_		,				lowercase_    )]
		_snake_case      			=			("{{:{}}} " * len(lowercase_    )).format(*lowercase_    )
		_snake_case      			=			[]
		lines.append(row_format.format(*lowercase_    )    )
		lines.append(row_format.format(*["-" * w for w in col_widths]    )    )
		for row in rows:
				lines.append(row_format.format(*lowercase_    )    )
		return "\n".join(lowercase_    )





class 	lowerCamelCase_				:




					'''simple docstring'''






					def __init__(       self,    __UpperCamelCase					):
							_snake_case      			=			args





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							print(
							    ANSI.red(
							        "ERROR! `huggingface-cli login` uses an outdated login mechanism "
							        "that is not compatible with the Hugging Face Hub backend anymore. "
							        "Please use `huggingface-cli login instead."					)					)





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							print(
							    ANSI.red(
							        "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
							        "`huggingface-cli whoami` instead."					)					)
							_snake_case      			=			HfFolder.get_token()
							if token is None:
									print("Not logged in"					)
									exit()
							try:
									_snake_case			,       _snake_case      			=			whoami(__UpperCamelCase					)
									print(__UpperCamelCase					)
									if orgs:
											print(ANSI.bold("orgs: "					),    ",".join(__UpperCamelCase					)					)
							except HTTPError as e:
									print(__UpperCamelCase					)
									print(ANSI.red(e.response.text					)					)
									exit(1					)





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							print(
							    ANSI.red(
							        "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
							        "that is not compatible with the Hugging Face Hub backend anymore. "
							        "Please use `huggingface-cli logout instead."					)					)





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							print(
							    ANSI.red(
							        "WARNING! Managing repositories through transformers-cli is deprecated. "
							        "Please use `huggingface-cli` instead."					)					)
							_snake_case      			=			HfFolder.get_token()
							if token is None:
									print("Not logged in"					)
									exit(1					)
							try:
									_snake_case      			=			subprocess.check_output(["git", "--version"]					).decode("utf-8"					)
									print(ANSI.gray(stdout.strip()					)					)
							except FileNotFoundError:
									print("Looks like you do not have git installed, please install."					)

							try:
									_snake_case      			=			subprocess.check_output(["git-lfs", "--version"]					).decode("utf-8"					)
									print(ANSI.gray(stdout.strip()					)					)
							except FileNotFoundError:
									print(
									    ANSI.red(
									        "Looks like you do not have git-lfs installed, please install."
									        " You can install from https://git-lfs.github.com/."
									        " Then run `git lfs install` (you only have to do this once)."					)					)
							print(""					)

							_snake_case			,       _snake_case      			=			whoami(__UpperCamelCase					)
							_snake_case      			=			self.args.organization if self.args.organization is not None else user
							_snake_case      			=			f'''{namespace}/{self.args.name}'''
							print(f'''You are about to create {ANSI.bold(__UpperCamelCase					)}'''					)

							if not self.args.yes:
									_snake_case      			=			input("Proceed? [Y/n] "					).lower()
									if not (choice == "" or choice == "y" or choice == "yes"):
											print("Abort"					)
											exit()
							try:
									_snake_case      			=			create_repo(__UpperCamelCase,    name=self.args.name,    organization=self.args.organization					)
							except HTTPError as e:
									print(__UpperCamelCase					)
									print(ANSI.red(e.response.text					)					)
									exit(1					)
							print("\nYour repo now lives at:"					)
							print(f'''  {ANSI.bold(__UpperCamelCase					)}'''					)
							print("\nYou can clone it locally with the command below, and commit/push as usual."					)
							print(f'''\n  git clone {url}'''					)
							print(""					)





