




'''simple docstring'''







import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir





@dataclass
class 	lowerCamelCase_				:




					'''simple docstring'''






					lowercase__       =					42





class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			get_tests_dir("fixtures/test_sentencepiece.model"					)
							_snake_case      			=			get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model"					)

							_snake_case      			=			FakeOriginalTokenizer(vocab_file=__UpperCamelCase					)

							with warnings.catch_warnings(record=__UpperCamelCase					) as w:
									_snake_case      			=			SpmConverter(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    0					)

							_snake_case      			=			FakeOriginalTokenizer(vocab_file=__UpperCamelCase					)

							with warnings.catch_warnings(record=__UpperCamelCase					) as w:
									_snake_case      			=			SpmConverter(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    1					)

							self.assertIn(
							    "The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option"
							    " which is not implemented in the fast tokenizers.",    str(w[0].message					),    )






"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import _LazyModule


SCREAMING_SNAKE_CASE__       =	{"tokenization_bert_japanese": ["BertJapaneseTokenizer", "CharacterTokenizer", "MecabTokenizer"]}


if TYPE_CHECKING:
					from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer

else:
					import sys

					SCREAMING_SNAKE_CASE__       =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


