
import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir







@dataclass
class 	lowerCamelCase						:
     UpperCAmelCase__							:				str







class 	lowerCamelCase						(    unittest.TestCase    ):





     def 						UpperCAmelCase(self :       Optional[int]  )		->		Union[str, Any]:
         snake_case						=   get_tests_dir("fixtures/test_sentencepiece.model"  )
         snake_case						=   get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model"  )

         snake_case						=   FakeOriginalTokenizer(vocab_file=_A  )

         with warnings.catch_warnings(record=_A  ) as w:
             snake_case						=   SpmConverter(_A  )
         self.assertEqual(len(_A  )							,       0  )

         snake_case						=   FakeOriginalTokenizer(vocab_file=_A  )

         with warnings.catch_warnings(record=_A  ) as w:
             snake_case						=   SpmConverter(_A  )
         self.assertEqual(len(_A  )							,       1  )

         self.assertIn(
             "The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option"
             " which is not implemented in the fast tokenizers."							,       str(w[0].message  )							,       )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A       =					logging.get_logger(__name__)

_A       =					{
    "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/config.json"
    # See all CPMAnt models at https://huggingface.co/models?filter=cpmant
}







class 	lowerCamelCase						(    A_    ):
     UpperCAmelCase__							:				List[Any] 	=						"cpmant"





     def __init__(self :       List[Any]							,       _A :       int = 3_0_7_2_0							,       _A :       int = 4_0_9_6							,       _A :       int = 3_2							,       _A :       int = 1_2_8							,       _A :       int = 1_0_2_4_0							,       _A :       int = 4_8							,       _A :       int = 0.0							,       _A :       int = 5_1_2							,       _A :       int = 2_0_4_8							,       _A :       int = 1E-6							,       _A :       float = 1.0							,       _A :       int = 3_2							,       _A :       int = 3_2							,       _A :       int = 3_2							,       _A :       bool = True							,       **_A :       Dict							,       )		->		Dict:
         super().__init__(**_A  )
         snake_case						=   prompt_types
         snake_case						=   prompt_length
         snake_case						=   segment_types
         snake_case						=   hidden_size
         snake_case						=   num_attention_heads
         snake_case						=   dim_head
         snake_case						=   dim_ff
         snake_case						=   num_hidden_layers
         snake_case						=   position_bias_num_buckets
         snake_case						=   position_bias_max_distance
         snake_case						=   dropout_p
         snake_case						=   eps
         snake_case						=   use_cache
         snake_case						=   vocab_size
         snake_case						=   init_std

