






"""simple docstring"""

import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch

class     A_				(   unittest.TestCase	):







				@patch("sys.argv"    ,     ["fakeprogrampath", "env"]						)
				def      _snake_case       (				self				:    Dict						)					->       List[str]:
										# test transformers-cli env
										import transformers.commands.transformers_cli

										with CaptureStd() as cs:
																transformers.commands.transformers_cli.main()
										self.assertIn("Python version"    ,     cs.out						)
										self.assertIn("Platform"    ,     cs.out						)
										self.assertIn("Using distributed or parallel set-up in script?"    ,     cs.out						)







				@is_pt_tf_cross_test
				@patch(
				    "sys.argv"    ,     ["fakeprogrampath", "pt-to-tf", "--model-name", "hf-internal-testing/tiny-random-gptj", "--no-pr"]						)
				def      _snake_case       (				self				:    Union[str, Any]						)					->       List[Any]:
										import transformers.commands.transformers_cli

										shutil.rmtree("/tmp/hf-internal-testing/tiny-random-gptj"    ,     ignore_errors=__lowerCamelCase						)  # cleans potential past runs
										transformers.commands.transformers_cli.main()

										# The original repo has no TF weights -- if they exist, they were created by the CLI
										self.assertTrue(os.path.exists("/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5"						)						)







				@require_torch
				@patch("sys.argv"    ,     ["fakeprogrampath", "download", "hf-internal-testing/tiny-random-gptj", "--cache-dir", "/tmp"]						)
				def      _snake_case       (				self				:    Any						)					->       Tuple:
										import transformers.commands.transformers_cli

										# # remove any previously downloaded model to start clean
										shutil.rmtree("/tmp/models--hf-internal-testing--tiny-random-gptj"    ,     ignore_errors=__lowerCamelCase						)

										# run the command
										transformers.commands.transformers_cli.main()

										# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
										self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/blobs"						)						)
										self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/refs"						)						)
										self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots"						)						)







				@require_torch
				@patch(
				    "sys.argv"    ,     [
				        "fakeprogrampath",
				        "download",
				        "hf-internal-testing/test_dynamic_model_with_tokenizer",
				        "--trust-remote-code",
				        "--cache-dir",
				        "/tmp",
				    ]    ,     )
				def      _snake_case       (				self				:    Union[str, Any]						)					->       List[str]:
										import transformers.commands.transformers_cli

										# # remove any previously downloaded model to start clean
										shutil.rmtree("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer"    ,     ignore_errors=__lowerCamelCase						)

										# run the command
										transformers.commands.transformers_cli.main()

										# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
										self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs"						)						)
										self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs"						)						)
										self.assertTrue(
										    os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots"						)						)











"""simple docstring"""

import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase

@require_tokenizers
class     A_				(   unittest.TestCase	):
				UpperCAmelCase__										=					EsmTokenizer







				def      _snake_case       (				self				:    List[str]						)					->       Dict:
										super().setUp()
										__magic_name__      =  tempfile.mkdtemp()
										# fmt: off
										__magic_name__      =  ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
										# fmt: on
										__magic_name__      =  os.path.join(self.tmpdirname    ,     VOCAB_FILES_NAMES["vocab_file"]						)
										with open(self.vocab_file    ,     "w"    ,     encoding="utf-8"						) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







				def      _snake_case       (				self				:    Optional[Any]    ,     **__lowerCamelCase				:    str						)					->       List[PreTrainedTokenizerBase]:
										return [self.get_tokenizer(**__lowerCamelCase						)]







				def      _snake_case       (				self				:    Dict    ,     **__lowerCamelCase				:    int						)					->       PreTrainedTokenizer:
										return self.tokenizer_class.from_pretrained(self.tmpdirname    ,     **__lowerCamelCase						)







				def      _snake_case       (				self				:    str						)					->       Optional[int]:
										__magic_name__      =  self.tokenizer_class(self.vocab_file						)

										__magic_name__      =  tokenizer.tokenize("LAGVS"						)
										self.assertListEqual(__lowerCamelCase    ,     ["L", "A", "G", "V", "S"]						)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)    ,     [4, 5, 6, 7, 8]						)







				def      _snake_case       (				self				:    int						)					->       Dict:
										__magic_name__      =  self.tokenizer_class(self.vocab_file						)

										__magic_name__      =  "LAGVS"
										self.assertListEqual(tokenizer.encode(__lowerCamelCase						)    ,     [0, 4, 5, 6, 7, 8, 2]						)







				def      _snake_case       (				self				:    List[Any]						)					->       Tuple:
										__magic_name__      =  self.tokenizer_class(self.vocab_file						)

										__magic_name__      =  ["LAGVS", "WCB"]
										__magic_name__      =  tokenizer(__lowerCamelCase    ,     padding=__lowerCamelCase						)["input_ids"]

										self.assertListEqual(__lowerCamelCase    ,     [[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2]]						)







				def      _snake_case       (				self				:    Union[str, Any]						)					->       List[str]:
										__magic_name__      =  self.tokenizer_class(self.vocab_file						)

										__magic_name__      =  ["LAGVS", "WCB"]
										__magic_name__      =  tokenizer(__lowerCamelCase    ,     padding=__lowerCamelCase						)["input_ids"]

										self.assertListEqual(__lowerCamelCase    ,     [[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2, 1, 1]]						)







				def      _snake_case       (				self				:    Union[str, Any]						)					->       Dict:
										__magic_name__      =  self.get_tokenizers(fast=__lowerCamelCase						)
										for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''						):
																						__magic_name__      =  "<unk>"
																						__magic_name__      =  "<mask>"

																						__magic_name__      =  tokenizer.tokenize(__lowerCamelCase						)
																						__magic_name__      =  tokenizer.tokenize(__lowerCamelCase						)

																						self.assertEqual(len(__lowerCamelCase						)    ,     1						)
																						self.assertEqual(len(__lowerCamelCase						)    ,     1						)
																						self.assertEqual(token_a[0]    ,     __lowerCamelCase						)
																						self.assertEqual(token_a[0]    ,     __lowerCamelCase						)



