

"""simple docstring"""




import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand





class      __UpperCamelCase					(  _A		):


		@staticmethod
		def    SCREAMING_SNAKE_CASE__  (__SCREAMING_SNAKE_CASE					:						ArgumentParser):
							A     =   parser.add_parser("login"	,	help="Log in using the same credentials as on huggingface.co")
							login_parser.set_defaults(func=lambda __SCREAMING_SNAKE_CASE: LoginCommand(__SCREAMING_SNAKE_CASE))
							A     =   parser.add_parser("whoami"	,	help="Find out which huggingface.co account you are logged in as.")
							whoami_parser.set_defaults(func=lambda __SCREAMING_SNAKE_CASE: WhoamiCommand(__SCREAMING_SNAKE_CASE))
							A     =   parser.add_parser("logout"	,	help="Log out")
							logout_parser.set_defaults(func=lambda __SCREAMING_SNAKE_CASE: LogoutCommand(__SCREAMING_SNAKE_CASE))

							# new system: git-based repo system
							A     =   parser.add_parser(
							    "repo"	,	help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos."	,	)
							A     =   repo_parser.add_subparsers(
							    help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands")
							A     =   repo_subparsers.add_parser(
							    "create"	,	help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co")
							repo_create_parser.add_argument(
							    "name"	,	type=__SCREAMING_SNAKE_CASE	,	help="Name for your model's repo. Will be namespaced under your username to build the model id."	,	)
							repo_create_parser.add_argument("--organization"	,	type=__SCREAMING_SNAKE_CASE	,	help="Optional: organization namespace.")
							repo_create_parser.add_argument("-y"	,	"--yes"	,	action="store_true"	,	help="Optional: answer Yes to the prompt")
							repo_create_parser.set_defaults(func=lambda __SCREAMING_SNAKE_CASE: RepoCreateCommand(__SCREAMING_SNAKE_CASE))





class      __UpperCamelCase					:
		SCREAMING_SNAKE_CASE														=       "\u001b[1m"
		SCREAMING_SNAKE_CASE														=       "\u001b[31m"
		SCREAMING_SNAKE_CASE														=       "\u001b[90m"
		SCREAMING_SNAKE_CASE														=       "\u001b[0m"


		@classmethod
		def    SCREAMING_SNAKE_CASE__  (cls					:						Union[str, Any]	,	__SCREAMING_SNAKE_CASE					:						Union[str, Any]):
							return F"""{cls._bold}{s}{cls._reset}"""


		@classmethod
		def    SCREAMING_SNAKE_CASE__  (cls					:						str	,	__SCREAMING_SNAKE_CASE					:						List[Any]):
							return F"""{cls._bold}{cls._red}{s}{cls._reset}"""


		@classmethod
		def    SCREAMING_SNAKE_CASE__  (cls					:						List[str]	,	__SCREAMING_SNAKE_CASE					:						Optional[int]):
							return F"""{cls._gray}{s}{cls._reset}"""

def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					A     =   [max(len(str(lowercase__ ) ) for x in col ) for col in zip(*lowercase__     ,				lowercase__ )]
					A     =   ("{{:{}}} " * len(lowercase__ )).format(*lowercase__ )
					A     =   []
					lines.append(row_format.format(*lowercase__ ) )
					lines.append(row_format.format(*["-" * w for w in col_widths] ) )
					for row in rows:
										lines.append(row_format.format(*lowercase__ ) )
					return "\n".join(lowercase__ )





class      __UpperCamelCase					:


		def __init__(self					:						str	,	__SCREAMING_SNAKE_CASE					:						Optional[int]):
							A     =   args





class      __UpperCamelCase					(  _A		):


		def    SCREAMING_SNAKE_CASE__  (self					:						Optional[int]):
							print(
							    ANSI.red(
							        "ERROR! `huggingface-cli login` uses an outdated login mechanism "
							        "that is not compatible with the Hugging Face Hub backend anymore. "
							        "Please use `huggingface-cli login instead."))





class      __UpperCamelCase					(  _A		):


		def    SCREAMING_SNAKE_CASE__  (self					:						Dict):
							print(
							    ANSI.red(
							        "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
							        "`huggingface-cli whoami` instead."))
							A     =   HfFolder.get_token()
							if token is None:
												print("Not logged in")
												exit()
							try:
												A     ,       A     =   whoami(__SCREAMING_SNAKE_CASE)
												print(__SCREAMING_SNAKE_CASE)
												if orgs:
																	print(ANSI.bold("orgs: ")	,	",".join(__SCREAMING_SNAKE_CASE))
							except HTTPError as e:
												print(__SCREAMING_SNAKE_CASE)
												print(ANSI.red(e.response.text))
												exit(1)





class      __UpperCamelCase					(  _A		):


		def    SCREAMING_SNAKE_CASE__  (self					:						List[Any]):
							print(
							    ANSI.red(
							        "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
							        "that is not compatible with the Hugging Face Hub backend anymore. "
							        "Please use `huggingface-cli logout instead."))





class      __UpperCamelCase					(  _A		):


		def    SCREAMING_SNAKE_CASE__  (self					:						Tuple):
							print(
							    ANSI.red(
							        "WARNING! Managing repositories through transformers-cli is deprecated. "
							        "Please use `huggingface-cli` instead."))
							A     =   HfFolder.get_token()
							if token is None:
												print("Not logged in")
												exit(1)
							try:
												A     =   subprocess.check_output(["git", "--version"]).decode("utf-8")
												print(ANSI.gray(stdout.strip()))
							except FileNotFoundError:
												print("Looks like you do not have git installed, please install.")

							try:
												A     =   subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
												print(ANSI.gray(stdout.strip()))
							except FileNotFoundError:
												print(
												    ANSI.red(
												        "Looks like you do not have git-lfs installed, please install."
												        " You can install from https://git-lfs.github.com/."
												        " Then run `git lfs install` (you only have to do this once)."))
							print("")

							A     ,       A     =   whoami(__SCREAMING_SNAKE_CASE)
							A     =   self.args.organization if self.args.organization is not None else user
							A     =   F"""{namespace}/{self.args.name}"""
							print(F"""You are about to create {ANSI.bold(__SCREAMING_SNAKE_CASE)}""")

							if not self.args.yes:
												A     =   input("Proceed? [Y/n] ").lower()
												if not (choice == "" or choice == "y" or choice == "yes"):
																	print("Abort")
																	exit()
							try:
												A     =   create_repo(__SCREAMING_SNAKE_CASE	,	name=self.args.name	,	organization=self.args.organization)
							except HTTPError as e:
												print(__SCREAMING_SNAKE_CASE)
												print(ANSI.red(e.response.text))
												exit(1)
							print("\nYour repo now lives at:")
							print(F"""  {ANSI.bold(__SCREAMING_SNAKE_CASE)}""")
							print("\nYou can clone it locally with the command below, and commit/push as usual.")
							print(F"""\n  git clone {url}""")
							print("")













'''simple docstring'''

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 				__magic_name__  (						unittest.TestCase					):







	"""simple docstring"""



	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Optional[int]   	=      list(tokenizer_class.max_model_input_sizes.keys()							)
							for model_name in sa_models[:1]:
													lowercase_   : List[str]   	=      tokenizer_class.from_pretrained(_lowercase							)
													self.assertIsNotNone(_lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)

													for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_lowercase	,   _lowercase							)
																			lowercase_   : Optional[int]   	=      tokenizer.convert_tokens_to_ids(_lowercase							)
																			self.assertIsInstance(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase = None							)    ->	str:
							lowercase_   : int   	=      pickle.dumps(_lowercase							)
							self.assertIsNotNone(_lowercase							)

							lowercase_   : Any   	=      pickle.loads(_lowercase							)

							# Ensure is_fast is correctly restored
							self.assertEqual(be_restored.is_fast	,   be_original.is_fast							)

							# Ensure encodings are potentially correctly restored
							if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings							)
							else:
													self.assertIsNone(be_restored.encodings							)

							# Ensure the keys are the same
							for original_v, restored_v in zip(be_original.values()	,   be_restored.values()							):
													if equal_op:
																			self.assertTrue(equal_op(_lowercase	,   _lowercase							)							)
													else:
																			self.assertEqual(_lowercase	,   _lowercase							)
	@slow
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							self.check_tokenizer_from_pretrained(_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							self.assertEqual(TensorType('tf'							)	,   TensorType.TENSORFLOW							)
							self.assertEqual(TensorType('pt'							)	,   TensorType.PYTORCH							)
							self.assertEqual(TensorType('np'							)	,   TensorType.NUMPY							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							import numpy as np

							lowercase_   : Any   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							# Python no tensor
							with self.subTest('BatchEncoding (Python, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)
	@require_tf
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							import tensorflow as tf

							def tf_array_equals(_lowercase	,   _lowercase							):
													return tf.reduce_all(tf.equal(_lowercase	,   _lowercase							)							)

							lowercase_   : Optional[Any]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)

							with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)
	@require_torch
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							import torch

							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Optional[int]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Tuple   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('Python Tokenizer'							):
													self.assertFalse(tokenizer_p('Small example to_encode'							).is_fast							)

							with self.subTest('Rust Tokenizer'							):
													self.assertTrue(tokenizer_r('Small example to_encode'							).is_fast							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : Optional[Any]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)
							lowercase_   : Dict   	=      tokenizer_r(['Test', '\xad', 'test']	,   is_split_into_words=_lowercase							)

							self.assertEqual(encoded.word_to_tokens(0							)	,   TokenSpan(start=1	,   end=2							)							)
							self.assertEqual(encoded.word_to_tokens(1							)	,   _lowercase							)
							self.assertEqual(encoded.word_to_tokens(2							)	,   TokenSpan(start=2	,   end=3							)							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Any   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : Union[str, Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[int]   	=      batch.convert_to_tensors(tensor_type='np'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Union[str, Any]   	=      batch.convert_to_tensors(tensor_type='pt'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : List[Any]   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Tuple   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='tf'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_flax
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : int   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : List[Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Any   	=      [{'input_ids': np.array([0, 1, 2]							)}, {'input_ids': np.array([0, 1, 2, 3]							)}]
							lowercase_   : str   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : List[str]   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='np'							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							import torch

							lowercase_   : Union[str, Any]   	=      [{'input_ids': torch.tensor([0, 1, 2]							)}, {'input_ids': torch.tensor([0, 1, 2, 3]							)}]
							lowercase_   : Optional[int]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : str   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Tuple   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='pt'							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Any:
							import tensorflow as tf

							lowercase_   : Any   	=      [{'input_ids': tf.constant([0, 1, 2]							)}, {'input_ids': tf.constant([0, 1, 2, 3]							)}]
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Dict   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='tf'							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[int]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							PreTrainedTokenizerFast(tokenizer_object=_lowercase							)







	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	int:
							lowercase_   : Union[str, Any]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(_lowercase	,   'tokenizer.json'							)							)
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase	,   'tokenizer.json'							)							)
