



"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowerCAmelCase_   						=   {
    'configuration_wav2vec2': ['WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Wav2Vec2Config'],
    'feature_extraction_wav2vec2': ['Wav2Vec2FeatureExtractor'],
    'processing_wav2vec2': ['Wav2Vec2Processor'],
    'tokenization_wav2vec2': ['Wav2Vec2CTCTokenizer', 'Wav2Vec2Tokenizer'],
}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'Wav2Vec2ForAudioFrameClassification',
							    'Wav2Vec2ForCTC',
							    'Wav2Vec2ForMaskedLM',
							    'Wav2Vec2ForPreTraining',
							    'Wav2Vec2ForSequenceClassification',
							    'Wav2Vec2ForXVector',
							    'Wav2Vec2Model',
							    'Wav2Vec2PreTrainedModel',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFWav2Vec2ForCTC',
							    'TFWav2Vec2Model',
							    'TFWav2Vec2PreTrainedModel',
							    'TFWav2Vec2ForSequenceClassification',
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'FlaxWav2Vec2ForCTC',
							    'FlaxWav2Vec2ForPreTraining',
							    'FlaxWav2Vec2Model',
							    'FlaxWav2Vec2PreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
							from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
							from .processing_wavaveca import WavaVecaProcessor
							from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_wavaveca import (
														    WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
														    WavaVecaForAudioFrameClassification,
														    WavaVecaForCTC,
														    WavaVecaForMaskedLM,
														    WavaVecaForPreTraining,
														    WavaVecaForSequenceClassification,
														    WavaVecaForXVector,
														    WavaVecaModel,
														    WavaVecaPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_wavaveca import (
														    TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFWavaVecaForCTC,
														    TFWavaVecaForSequenceClassification,
														    TFWavaVecaModel,
														    TFWavaVecaPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_wavaveca import (
														    FlaxWavaVecaForCTC,
														    FlaxWavaVecaForPreTraining,
														    FlaxWavaVecaModel,
														    FlaxWavaVecaPreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase_   						=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)







import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				List[str]							=    CpmAntTokenizer
      A_						:				Optional[int]							=    False


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         super().setUp()

         __snake_case							       =   [
             '''<d>''',
             '''</d>''',
             '''<s>''',
             '''</s>''',
             '''</_>''',
             '''<unk>''',
             '''<pad>''',
             '''</n>''',
             '''我''',
             '''是''',
             '''C''',
             '''P''',
             '''M''',
             '''A''',
             '''n''',
             '''t''',
         ]
         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''vocab_file''']      )
         with open(self.vocab_file				,			'''w'''				,			encoding='''utf-8'''      ) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )


      @tooslow
      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   ['''今天''', '''天气''', '''真''', '''好''', '''！''']
         __snake_case							       =   tokenizer.tokenize(a__      )
         self.assertListEqual(a__				,			a__      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   [tokenizer.bos_token] + tokens

         __snake_case							       =   [6, 9802, 1_4962, 2082, 831, 244]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )				,			a__      )

         __snake_case							       =   tokenizer.decode(a__      )
         self.assertEqual(a__				,			a__      )

