






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


A_       :Optional[int]       =						logging.get_logger(__name__)

A_       :List[str]       =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A_       :str       =						{
    '''vocab_file''': {'''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'''},
    '''tokenizer_file''': {
        '''mobilebert-uncased''': '''https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'''
    },
}

A_       :Tuple       =						{'''mobilebert-uncased''': 512}


A_       :Optional[Any]       =						{}




class 				__A     (			snake_case_ ):





				"""simple docstring"""







				UpperCamelCase__ :      Optional[Any]			 =VOCAB_FILES_NAMES
				UpperCamelCase__ :      List[str]			 =PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase__ :      Tuple			 =PRETRAINED_INIT_CONFIGURATION
				UpperCamelCase__ :      str			 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase__ :      int			 =MobileBertTokenizer


				def __init__( self							,  lowerCamelCase__=None							,  lowerCamelCase__=None							,  lowerCamelCase__=True							,  lowerCamelCase__="[UNK]"							,  lowerCamelCase__="[SEP]"							,  lowerCamelCase__="[PAD]"							,  lowerCamelCase__="[CLS]"							,  lowerCamelCase__="[MASK]"							,  lowerCamelCase__=True							,  lowerCamelCase__=None							,  **lowerCamelCase__							,  ):




									"""simple docstring"""

									super().__init__(
									    lowerCamelCase__							,  tokenizer_file=lowerCamelCase__							,  do_lower_case=lowerCamelCase__							,  unk_token=lowerCamelCase__							,  sep_token=lowerCamelCase__							,  pad_token=lowerCamelCase__							,  cls_token=lowerCamelCase__							,  mask_token=lowerCamelCase__							,  tokenize_chinese_chars=lowerCamelCase__							,  strip_accents=lowerCamelCase__							,  **lowerCamelCase__							,  )

									__UpperCamelCase      :					Dict       		=json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
									if (
									    normalizer_state.get('lowercase'							,  lowerCamelCase__    ) != do_lower_case
									    or normalizer_state.get('strip_accents'							,  lowerCamelCase__    ) != strip_accents
									    or normalizer_state.get('handle_chinese_chars'							,  lowerCamelCase__    ) != tokenize_chinese_chars
									):
														__UpperCamelCase      :					Optional[int]       		=getattr(lowerCamelCase__							,  normalizer_state.pop('type'    )    )
														__UpperCamelCase      :					Union[str, Any]       		=do_lower_case
														__UpperCamelCase      :					List[Any]       		=strip_accents
														__UpperCamelCase      :					List[str]       		=tokenize_chinese_chars
														__UpperCamelCase      :					Tuple       		=normalizer_class(**lowerCamelCase__    )

									__UpperCamelCase      :					Any       		=do_lower_case


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__=None    ):




									"""simple docstring"""

									__UpperCamelCase      :					Dict       		=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

									return output


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=[self.sep_token_id]
									__UpperCamelCase      :					Tuple       		=[self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep    ) * [0]
									return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=self._tokenizer.model.save(lowerCamelCase__							,  name=lowerCamelCase__    )
									return tuple(lowerCamelCase__    )



# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__A				='''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
			raise ImportWarning(
			    '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
			)

if version.parse(pyarrow.__version__).major < 8:
			raise ImportWarning(
			    '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
			    '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
			)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__A				=concatenate_datasets
__A				=DownloadConfig
__A				=DownloadManager
__A				=DownloadMode
__A				=DownloadConfig
__A				=DownloadMode
__A				=DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


