



'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__:   Optional[int]								=			logging.get_logger(__name__)

UpperCamelCase__:   List[str]								=			{"vocab_file": "sentencepiece.model"}

UpperCamelCase__:   Tuple								=			{
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
}

UpperCamelCase__:   Optional[int]								=			{
    "google/rembert": 256,
}

class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	lowerCamelCase__			 =       VOCAB_FILES_NAMES
	lowerCamelCase__			 =       PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__			 =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







	def __init__(						self			:     Union[str, Any]		,			__snake_case			:     str		,			__snake_case			:     Optional[int]=False		,			__snake_case			:     Tuple=True		,			__snake_case			:     int=True		,			__snake_case			:     str="[CLS]"		,			__snake_case			:     Tuple="[SEP]"		,			__snake_case			:     str="[UNK]"		,			__snake_case			:     Optional[Any]="[SEP]"		,			__snake_case			:     Union[str, Any]="[PAD]"		,			__snake_case			:     Dict="[CLS]"		,			__snake_case			:     Optional[Any]="[MASK]"		,			**__snake_case			:     List[Any]		,			)			->		int:
								super().__init__(
								    do_lower_case=__snake_case		,			remove_space=__snake_case		,			keep_accents=__snake_case		,			bos_token=__snake_case		,			eos_token=__snake_case		,			unk_token=__snake_case		,			sep_token=__snake_case		,			pad_token=__snake_case		,			cls_token=__snake_case		,			mask_token=__snake_case		,			**__snake_case		,			)

								UpperCAmelCase :       Tuple 							=		do_lower_case
								UpperCAmelCase :       Dict 							=		remove_space
								UpperCAmelCase :       Any 							=		keep_accents
								UpperCAmelCase :       Union[str, Any] 							=		vocab_file

								UpperCAmelCase :       int 							=		spm.SentencePieceProcessor()
								self.sp_model.Load(__snake_case		)







	@property
	def      A	(						self			:     Union[str, Any]		)			->		Optional[int]:
								return len(self.sp_model		)







	def      A	(						self			:     Tuple		)			->		Optional[Any]:
								UpperCAmelCase :       Optional[int] 							=		{self.convert_ids_to_tokens(__snake_case		): i for i in range(self.vocab_size		)}
								vocab.update(self.added_tokens_encoder		)
								return vocab







	def __getstate__(						self			:     List[str]		)			->		Optional[Any]:
								UpperCAmelCase :       List[Any] 							=		self.__dict__.copy()
								UpperCAmelCase :       Dict 							=		None
								return state







	def __setstate__(						self			:     Any		,			__snake_case			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		d
								UpperCAmelCase :       Optional[int] 							=		spm.SentencePieceProcessor()
								self.sp_model.Load(self.vocab_file		)







	def      A	(						self			:     Tuple		,			__snake_case			:     Any		,			__snake_case			:     Union[str, Any]=False		)			->		Union[str, Any]:
								UpperCAmelCase :       int 							=		self.sp_model.EncodeAsPieces(__snake_case		)
								return pieces







	def      A	(						self			:     str		,			__snake_case			:     Union[str, Any]		)			->		int:
								return self.sp_model.PieceToId(__snake_case		)







	def      A	(						self			:     str		,			__snake_case			:     str		)			->		List[Any]:
								return self.sp_model.IdToPiece(__snake_case		)







	def      A	(						self			:     int		,			__snake_case			:     Any		)			->		List[Any]:
								UpperCAmelCase :       Optional[int] 							=		self.sp_model.decode_pieces(__snake_case		)
								return out_string







	def      A	(						self			:     Dict		,			__snake_case			:     List[int]		,			__snake_case			:     Optional[List[int]] = None		)			->		List[int]:
								UpperCAmelCase :       List[Any] 							=		[self.sep_token_id]
								UpperCAmelCase :       Union[str, Any] 							=		[self.cls_token_id]
								if token_ids_a is None:
															return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     List[int]		,			__snake_case			:     Optional[List[int]] = None		,			__snake_case			:     bool = False		)			->		List[int]:

								if already_has_special_tokens:
															if token_ids_a is not None:
																						raise ValueError(
																						    '''You should not supply a second sequence if the provided sequence of '''
																						    '''ids is already formatted with special tokens for the model.'''		)
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
															return [1] + ([0] * len(__snake_case		)) + [1] + ([0] * len(__snake_case		)) + [1]
								return [1] + ([0] * len(__snake_case		)) + [1]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     List[int]		,			__snake_case			:     Optional[List[int]] = None		)			->		List[int]:
								UpperCAmelCase :       str 							=		[self.sep_token_id]
								UpperCAmelCase :       Tuple 							=		[self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep		) * [0]
								return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      A	(						self			:     int		,			__snake_case			:     str		,			__snake_case			:     Optional[str] = None		)			->		Tuple[str]:
								if not os.path.isdir(__snake_case		):
															logger.error('''Vocabulary path ({}) should be a directory'''.format(__snake_case		)		)
															return
								UpperCAmelCase :       str 							=		os.path.join(
								    __snake_case		,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

								if os.path.abspath(self.vocab_file		) != os.path.abspath(__snake_case		):
															copyfile(self.vocab_file		,			__snake_case		)

								return (out_vocab_file,)










'''simple docstring'''






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Optional[Any]  )      ->   Optional[int]:
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 4
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_1", "col_2", "col_3"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,      [False, True]  )
def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Optional[int]  )      ->   Dict:
							UpperCAmelCase :       Dict 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[Any] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCAmelCase :       str 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase   ,      keep_in_memory=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						int  )      ->   Optional[int]:
							UpperCAmelCase :       Any 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Tuple 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Any 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       List[Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Dict 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						List[Any]   ,      _lowerCAmelCase  :						Tuple  )      ->   Tuple:
							UpperCAmelCase :       Optional[Any] 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Optional[int] 							=		{'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
							UpperCAmelCase :       int 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       Any 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_3", "col_1", "col_2"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Dict  )      ->   Union[str, Any]:
							# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
							UpperCAmelCase :       Tuple 							=		{'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
							UpperCAmelCase :       List[str] 							=		features.copy()
							UpperCAmelCase :       Union[str, Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[str] 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_2", "col_3", "col_1"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''split'''   ,      [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def  snake_case_     (					_lowerCAmelCase  :						Union[str, Any]   ,      _lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						List[str]  )      ->   Optional[Any]:
							UpperCAmelCase :       Any 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[Any] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       List[Any] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase   ,      split=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.split == split if split else "train"


@pytest.mark.parametrize('''path_type'''   ,      [str, list]  )
def  snake_case_     (					_lowerCAmelCase  :						List[Any]   ,      _lowerCAmelCase  :						str   ,      _lowerCAmelCase  :						Any  )      ->   Dict:
							if issubclass(_lowerCAmelCase   ,      _lowerCAmelCase  ):
														UpperCAmelCase :       str 							=		jsonl_path
							elif issubclass(_lowerCAmelCase   ,      _lowerCAmelCase  ):
														UpperCAmelCase :       Dict 							=		[jsonl_path]
							UpperCAmelCase :       int 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Dict 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[int] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Dict=("train",)  )      ->   Union[str, Any]:
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							for split in splits:
														UpperCAmelCase :       List[str] 							=		dataset_dict[split]
														assert dataset.num_rows == 4
														assert dataset.num_columns == 3
														assert dataset.column_names == ["col_1", "col_2", "col_3"]
														for feature, expected_dtype in expected_features.items():
																					assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,      [False, True]  )
def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						str  )      ->   Any:
							UpperCAmelCase :       Optional[Any] 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[str] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCAmelCase :       Optional[int] 							=		JsonDatasetReader({'''train''': jsonl_path}   ,      cache_dir=_lowerCAmelCase   ,      keep_in_memory=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Union[str, Any]  )      ->   int:
							UpperCAmelCase :       Dict 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Dict 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[int] 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       Union[str, Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		JsonDatasetReader({'''train''': jsonl_path}   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize('''split'''   ,      [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def  snake_case_     (					_lowerCAmelCase  :						Any   ,      _lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Dict  )      ->   Union[str, Any]:
							if split:
														UpperCAmelCase :       Optional[int] 							=		{split: jsonl_path}
							else:
														UpperCAmelCase :       Any 							=		'''train'''
														UpperCAmelCase :       Any 							=		{'''train''': jsonl_path, '''test''': jsonl_path}
							UpperCAmelCase :       Tuple 							=		tmp_path / '''cache'''
							UpperCAmelCase :       int 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[Any] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase   ,      splits=list(path.keys()  )  )
							assert all(dataset[split].split == split for split in path.keys()  )


def  snake_case_     (					_lowerCAmelCase  :						List[str]  )      ->   str:
							return json.load(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Dict  )      ->   str:
							return [json.loads(_lowerCAmelCase  ) for line in buffer]




class 				SCREAMING_SNAKE_CASE:







	"""simple docstring"""






	@pytest.mark.parametrize('''lines, load_json_function'''		,			[(True, load_json_lines), (False, load_json)]		)
	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     Tuple		,			__snake_case			:     Optional[Any]		,			__snake_case			:     Optional[int]		)			->		Dict:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Union[str, Any] 							=		load_json_function(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								assert isinstance(exported_content[0]		,			__snake_case		)
								assert len(__snake_case		) == 10







	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''		,			[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''		), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]		,			)
	def      A	(						self			:     Optional[int]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     str		,			__snake_case			:     Optional[Any]		,			__snake_case			:     List[str]		,			__snake_case			:     Optional[Any]		)			->		List[Any]:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			orient=__snake_case		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Union[str, Any] 							=		load_json(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								if keys:
															if container is dict:
																						assert exported_content.keys() == keys
															else:
																						assert exported_content[0].keys() == keys
								else:
															assert not hasattr(__snake_case		,			'''keys'''		) and not hasattr(exported_content[0]		,			'''keys'''		)
								if len_at:
															assert len(exported_content[len_at]		) == 10
								else:
															assert len(__snake_case		) == 10







	@pytest.mark.parametrize('''lines, load_json_function'''		,			[(True, load_json_lines), (False, load_json)]		)
	def      A	(						self			:     str		,			__snake_case			:     str		,			__snake_case			:     str		,			__snake_case			:     int		)			->		Any:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			num_proc=2		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Any 							=		load_json_function(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								assert isinstance(exported_content[0]		,			__snake_case		)
								assert len(__snake_case		) == 10







	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''		,			[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''		), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]		,			)
	def      A	(						self			:     Any		,			__snake_case			:     int		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Optional[int]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     List[str]		)			->		Any:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			orient=__snake_case		,			num_proc=2		).write()
															buffer.seek(0		)
															UpperCAmelCase :       List[str] 							=		load_json(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								if keys:
															if container is dict:
																						assert exported_content.keys() == keys
															else:
																						assert exported_content[0].keys() == keys
								else:
															assert not hasattr(__snake_case		,			'''keys'''		) and not hasattr(exported_content[0]		,			'''keys'''		)
								if len_at:
															assert len(exported_content[len_at]		) == 10
								else:
															assert len(__snake_case		) == 10







	def      A	(						self			:     List[Any]		,			__snake_case			:     str		)			->		Dict:
								with pytest.raises(__snake_case		):
															with io.BytesIO() as buffer:
																						JsonDatasetWriter(__snake_case		,			__snake_case		,			num_proc=0		)







	@pytest.mark.parametrize('''compression, extension'''		,			[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]		)
	def      A	(						self			:     Optional[int]		,			__snake_case			:     Any		,			__snake_case			:     str		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Dict		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		tmp_path_factory.mktemp('''data'''		) / F"""test.json.{extension}"""
								UpperCAmelCase :       List[Any] 							=		str(shared_datadir / F"""test_file.json.{extension}"""		)
								JsonDatasetWriter(__snake_case		,			__snake_case		,			compression=__snake_case		).write()

								with fsspec.open(__snake_case		,			'''rb'''		,			compression='''infer'''		) as f:
															UpperCAmelCase :       str 							=		f.read()
								with fsspec.open(__snake_case		,			'''rb'''		,			compression='''infer'''		) as f:
															UpperCAmelCase :       Optional[int] 							=		f.read()
								assert exported_content == original_content





