"""simple docstring"""
import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir



@dataclass
class 	lowerCAmelCase_     :



				"""simple docstring"""
				_lowerCAmelCase  :     str





class 	lowerCAmelCase_     (      unittest.TestCase       ):



				"""simple docstring"""

				def 		snake_case			(					self     ):






										"""simple docstring"""

										snake_case             =      get_tests_dir('fixtures/test_sentencepiece.model'     )
										snake_case             =      get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'     )

										snake_case             =      FakeOriginalTokenizer(vocab_file=lowerCAmelCase     )

										with warnings.catch_warnings(record=lowerCAmelCase     ) as w:
																snake_case             =      SpmConverter(lowerCAmelCase     )
										self.assertEqual(len(lowerCAmelCase     )		,     0     )

										snake_case             =      FakeOriginalTokenizer(vocab_file=lowerCAmelCase     )

										with warnings.catch_warnings(record=lowerCAmelCase     ) as w:
																snake_case             =      SpmConverter(lowerCAmelCase     )
										self.assertEqual(len(lowerCAmelCase     )		,     1     )

										self.assertIn(
										    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
										    ' which is not implemented in the fast tokenizers.'		,     str(w[0].message     )		,     )



"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import _LazyModule


SCREAMING_SNAKE_CASE__       =	{"tokenization_bert_japanese": ["BertJapaneseTokenizer", "CharacterTokenizer", "MecabTokenizer"]}


if TYPE_CHECKING:
					from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer

else:
					import sys

					SCREAMING_SNAKE_CASE__       =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


