






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_       :Optional[int]       =						'''▁'''

A_       :Dict       =						{'''vocab_file''': '''spiece.model'''}

A_       :str       =						{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

A_       :List[Any]       =						{
    '''google/pegasus-xsum''': 512,
}


A_       :Tuple       =						logging.get_logger(__name__)




class 				__A     (			snake_case_ ):





				"""simple docstring"""







				UpperCamelCase__ :      List[Any]			 =VOCAB_FILES_NAMES

				UpperCamelCase__ :      Tuple			 =VOCAB_FILES_NAMES
				UpperCamelCase__ :      Any			 =PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase__ :      str			 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase__ :      Dict			 =["""input_ids""", """attention_mask"""]


				def __init__( self							,  lowerCamelCase__							,  lowerCamelCase__="<pad>"							,  lowerCamelCase__="</s>"							,  lowerCamelCase__="<unk>"							,  lowerCamelCase__="<mask_2>"							,  lowerCamelCase__="<mask_1>"							,  lowerCamelCase__=None							,  lowerCamelCase__=103							,  lowerCamelCase__ = None							,  **lowerCamelCase__							,  ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=offset
									if additional_special_tokens is not None:
														if not isinstance(lowerCamelCase__							,  lowerCamelCase__    ):
																			raise TypeError(
																			    f'additional_special_tokens should be of type {type(lowerCamelCase__    )}, but is'
																			    f' {type(lowerCamelCase__    )}'    )

														__UpperCamelCase      :					Union[str, Any]       		=(
														    ([mask_token_sent] + additional_special_tokens)
														    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
														    else additional_special_tokens
														)
														# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
														additional_special_tokens_extended += [
														    f'<unk_{i}>' for i in range(len(lowerCamelCase__    )							,  self.offset - 1    )
														]

														if len(set(lowerCamelCase__    )    ) != len(lowerCamelCase__    ):
																			raise ValueError(
																			    'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
																			    f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'    )
														__UpperCamelCase      :					int       		=additional_special_tokens_extended
									else:
														__UpperCamelCase      :					str       		=[mask_token_sent] if mask_token_sent is not None else []
														additional_special_tokens += [f'<unk_{i}>' for i in range(2							,  self.offset    )]

									__UpperCamelCase      :					List[Any]       		={} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    eos_token=lowerCamelCase__							,  unk_token=lowerCamelCase__							,  mask_token=lowerCamelCase__							,  pad_token=lowerCamelCase__							,  mask_token_sent=lowerCamelCase__							,  offset=lowerCamelCase__							,  additional_special_tokens=lowerCamelCase__							,  sp_model_kwargs=self.sp_model_kwargs							,  **lowerCamelCase__							,  )
									__UpperCamelCase      :					str       		=mask_token_sent
									__UpperCamelCase      :					Tuple       		=vocab_file
									__UpperCamelCase      :					Optional[Any]       		=spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(lowerCamelCase__    )

									# add special tokens to encoder dict
									__UpperCamelCase      :					Tuple       		={
									    0: self.pad_token,
									    1: self.eos_token,
									}

									if self.mask_token_sent is not None:
														self.encoder.update(
														    {
														        2: self.mask_token_sent,
														        3: self.mask_token,
														    }    )

									if self.offset > 0:
														# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
														# mask_token_sent is already added to list -> so start at 1
														self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1							,  self.offset - 1    )}    )

									__UpperCamelCase      :					Union[str, Any]       		={v: k for k, v in self.encoder.items()}


				@property
				def       __lowercase							( self    ):




									"""simple docstring"""

									return len(self.sp_model    ) + self.offset


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		={self.convert_ids_to_tokens(lowerCamelCase__    ): i for i in range(self.vocab_size    )}
									vocab.update(self.added_tokens_encoder    )
									return vocab


				def __getstate__( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=self.__dict__.copy()
									__UpperCamelCase      :					Tuple       		=None
									return state


				def __setstate__( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=d

									# for backward compatibility
									if not hasattr(self							,  'sp_model_kwargs'    ):
														__UpperCamelCase      :					Optional[int]       		={}

									__UpperCamelCase      :					Optional[int]       		=spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(self.vocab_file    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									return self.sp_model.encode(lowerCamelCase__							,  out_type=lowerCamelCase__    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									if token in self.decoder:
														return self.decoder[token]
									elif token in self.added_tokens_decoder:
														return self.added_tokens_decoder[token]
									__UpperCamelCase      :					List[Any]       		=self.sp_model.piece_to_id(lowerCamelCase__    )
									return sp_id + self.offset


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									if index in self.encoder:
														return self.encoder[index]
									elif index in self.added_tokens_encoder:
														return self.added_tokens_encoder[index]
									else:
														__UpperCamelCase      :					int       		=self.sp_model.IdToPiece(index - self.offset    )
									return token


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=[]
									__UpperCamelCase      :					List[str]       		=''
									for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																			out_string += self.sp_model.decode(lowerCamelCase__    ) + token
																			__UpperCamelCase      :					Dict       		=[]
														else:
																			current_sub_tokens.append(lowerCamelCase__    )
									out_string += self.sp_model.decode(lowerCamelCase__    )
									return out_string.strip()


				def       __lowercase							( self							,  lowerCamelCase__=False    ):




									"""simple docstring"""

									return 1


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		=set(self.all_special_ids    )  # call it once instead of inside list comp
									all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

									return [1 if x in all_special_ids else 0 for x in seq]


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None							,  lowerCamelCase__ = False    ):




									"""simple docstring"""

									if already_has_special_tokens:
														return self._special_token_mask(lowerCamelCase__    )
									elif token_ids_a is None:
														return self._special_token_mask(lowerCamelCase__    ) + [1]
									else:
														return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__=None    ):




									"""simple docstring"""

									if token_ids_a is None:
														return token_ids_a + [self.eos_token_id]
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return token_ids_a + token_ids_a + [self.eos_token_id]







				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									if not os.path.isdir(lowerCamelCase__    ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
														return
									__UpperCamelCase      :					List[str]       		=os.path.join(
									    lowerCamelCase__							,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

									if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase__    ) and os.path.isfile(self.vocab_file    ):
														copyfile(self.vocab_file							,  lowerCamelCase__    )
									elif not os.path.isfile(self.vocab_file    ):
														with open(lowerCamelCase__							,  'wb'    ) as fi:
																			__UpperCamelCase      :					List[Any]       		=self.sp_model.serialized_model_proto()
																			fi.write(lowerCamelCase__    )

									return (out_vocab_file,)



# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

__A				='''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
			raise ImportWarning(
			    '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
			)

if version.parse(pyarrow.__version__).major < 8:
			raise ImportWarning(
			    '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
			    '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
			)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

__A				=concatenate_datasets
__A				=DownloadConfig
__A				=DownloadManager
__A				=DownloadMode
__A				=DownloadConfig
__A				=DownloadMode
__A				=DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


