






import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 				__A     (			snake_case_      ,     unittest.TestCase ):





				"""simple docstring"""







				UpperCamelCase__ :      Optional[int]			 =PhobertTokenizer
				UpperCamelCase__ :      str			 =False


				def       __lowercase							( self    ):




									"""simple docstring"""

									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									__UpperCamelCase      :					Dict       		=['T@@', 'i', 'I', 'R@@', 'r', 'e@@']
									__UpperCamelCase      :					List[Any]       		=dict(zip(lowerCamelCase__							,  range(len(lowerCamelCase__    )    )    )    )
									__UpperCamelCase      :					Any       		=['#version: 0.2', 'l à</w>']
									__UpperCamelCase      :					Tuple       		={'unk_token': '<unk>'}

									__UpperCamelCase      :					Optional[Any]       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['vocab_file']    )
									__UpperCamelCase      :					Union[str, Any]       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['merges_file']    )

									with open(self.vocab_file							,  'w'							,  encoding='utf-8'    ) as fp:
														for token in vocab_tokens:
																			fp.write(f'{token} {vocab_tokens[token]}\n'    )
									with open(self.merges_file							,  'w'							,  encoding='utf-8'    ) as fp:
														fp.write('\n'.join(lowerCamelCase__    )    )


				def       __lowercase							( self							,  **lowerCamelCase__    ):




									"""simple docstring"""

									kwargs.update(self.special_tokens_map    )
									return PhobertTokenizer.from_pretrained(self.tmpdirname							,  **lowerCamelCase__    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		='Tôi là VinAI Research'
									__UpperCamelCase      :					Union[str, Any]       		='T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
									return input_text, output_text







				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=PhobertTokenizer(self.vocab_file							,  self.merges_file							,  **self.special_tokens_map    )
									__UpperCamelCase      :					List[Any]       		='Tôi là VinAI Research'
									__UpperCamelCase      :					Optional[Any]       		='T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
									__UpperCamelCase      :					str       		=tokenizer.tokenize(lowerCamelCase__    )
									print(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__							,  lowerCamelCase__    )

									__UpperCamelCase      :					Dict       		=tokens + [tokenizer.unk_token]

									__UpperCamelCase      :					List[str]       		=[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )							,  lowerCamelCase__    )



# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__A				='''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
			raise ImportWarning(
			    '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
			)

if version.parse(pyarrow.__version__).major < 8:
			raise ImportWarning(
			    '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
			    '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
			)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__A				=concatenate_datasets
__A				=DownloadConfig
__A				=DownloadManager
__A				=DownloadMode
__A				=DownloadConfig
__A				=DownloadMode
__A				=DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


