

import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class   __lowercase				( UpperCAmelCase_	):




			"""simple docstring"""







			def        _SCREAMING_SNAKE_CASE			( self  :     int):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.array(TypedSequence([1, 2, 3]))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([1, 2, 3])   ,						type=pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									with self.assertRaises(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence([1, 2, 3]   ,						try_type=Value("bool")   ,						type=Value("int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence([1, 2, 3]   ,						type=Value("int32")))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
															SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						type=Value("int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([1, 2, 3]   ,						try_type=Value("int32")))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence(["foo", "bar"]   ,						try_type=Value("int64")))
									self.assertEqual(arr.type   ,						pa.string())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: int  		=       pa.array(TypedSequence([[[1, 2, 3]]]   ,						type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						ArrayaDExtensionType((1, 3)   ,						"int64"))



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
															SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						type=ArrayaD((1, 3)   ,						"int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([[[1, 2, 3]]]   ,						try_type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						ArrayaDExtensionType((1, 3)   ,						"int64"))



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: List[Any]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						try_type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						pa.string())




			@require_pil
			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									import PIL.Image

									SCREAMING_SNAKE_CASE_: Tuple  		=       PIL.Image.fromarray(np.arange(10   ,						dtype=np.uinta).reshape(2   ,						5))
									with patch(
									    "datasets.arrow_writer.cast_to_python_objects"   ,						side_effect=lowerCAmelCase__) as mock_cast_to_python_objects:
															SCREAMING_SNAKE_CASE_: Tuple  		=       pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]   ,						type=Image()))
															SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[int]  		=       mock_cast_to_python_objects.call_args_list[-1]
															self.assertIn("optimize_list_casting"   ,						lowerCAmelCase__)
															self.assertFalse(kwargs["optimize_list_casting"])







def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Dict  		=       pa.BufferReader(_UpperCAmelCase    ) if isinstance(_UpperCAmelCase				,	pa.Buffer    ) else pa.memory_map(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.ipc.open_stream(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       f.read_all()
						assert len(pa_table.to_batches()    ) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: str  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: int  		=       Features({"labels": ClassLabel(names=["neg", "pos"]    )}    )
						with ArrowWriter(stream=_UpperCAmelCase				,	features=_UpperCAmelCase    ) as writer:
												writer.write({"labels": 0}    )
												writer.write({"labels": 1}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.ipc.open_stream(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       f.read_all()
						SCREAMING_SNAKE_CASE_: str  		=       pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(_UpperCAmelCase    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: str  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												with pytest.raises(_UpperCAmelCase    ):
																		writer.write({"col_1": "foo", "col_2": 1}				,	key=[1, 2]    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												with pytest.raises(_UpperCAmelCase    ):
																		writer.write({"col_1": "foo", "col_2": 1}				,	key=10    )
																		writer.write({"col_1": "bar", "col_2": 2}				,	key=10    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Tuple  		=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												writer.write({"col_1": "foo", "col_2": 1}				,	key=1    )
												writer.write({"col_1": "bar", "col_2": 2}				,	key=2    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: List[Any]  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )
												writer.write_batch({"col_1": [], "col_2": []}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: List[str]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: str  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: Dict  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: Any  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}    )    )
												writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}    )    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: List[str]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







def 			A_      (    ):
						with tempfile.TemporaryDirectory() as tmp_dir:
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
												SCREAMING_SNAKE_CASE_: Any  		=       os.path.join(_UpperCAmelCase				,	"test.arrow"    )
												with ArrowWriter(path=_UpperCAmelCase				,	schema=pa.schema(_UpperCAmelCase    )    ) as writer:
																		writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: List[Any]  		=       writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
												_check_output(_UpperCAmelCase				,	1    )







def 			A_      (    _UpperCAmelCase    ):
						if pa.types.is_list(_UpperCAmelCase    ):
												return get_base_dtype(arr_type.value_type    )
						else:
												return arr_type







def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						if isinstance(lst[0]				,	_UpperCAmelCase    ):
												change_first_primitive_element_in_list(lst[0]				,	_UpperCAmelCase    )
						else:
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       value







@pytest.mark.parametrize("optimized_int_type, expected_dtype"				,	[(None, pa.intaa()), (Value("int32"    ), pa.intaa())]    )
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.array(TypedSequence(_UpperCAmelCase				,	optimized_int_type=_UpperCAmelCase    )    )
						assert get_base_dtype(arr.type    ) == expected_dtype







@pytest.mark.parametrize(
    "col, expected_dtype"				,	[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]				,	)
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase				,	_UpperCAmelCase    ):
						# in range
						SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(OptimizedTypedSequence(_UpperCAmelCase				,	col=_UpperCAmelCase    )    )
						assert get_base_dtype(arr.type    ) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       copy.deepcopy(_UpperCAmelCase    )
												SCREAMING_SNAKE_CASE_: str  		=       np.iinfo(expected_dtype.to_pandas_dtype()    ).max + 1
												change_first_primitive_element_in_list(_UpperCAmelCase				,	_UpperCAmelCase    )
												SCREAMING_SNAKE_CASE_: int  		=       pa.array(OptimizedTypedSequence(_UpperCAmelCase				,	col=_UpperCAmelCase    )    )
												assert get_base_dtype(arr.type    ) == pa.intaa()







@pytest.mark.parametrize("raise_exception"				,	[False, True]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Any  		=       str(tmp_path / "dataset-train.arrow"    )
						try:
												with ArrowWriter(path=_UpperCAmelCase    ) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed







def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: str  		=       "mock://dataset-train.arrow"
						with ArrowWriter(path=_UpperCAmelCase				,	storage_options=mockfs.storage_options    ) as writer:
												assert isinstance(writer._fs				,	type(_UpperCAmelCase    )    )
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Any  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(_UpperCAmelCase    )







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: List[str]  		=       pa.BufferOutputStream()
						with ParquetWriter(stream=_UpperCAmelCase    ) as writer:
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Dict  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       pq.read_table(_UpperCAmelCase    )
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}







@require_pil
@pytest.mark.parametrize("embed_local_files"				,	[False, True]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						import PIL.Image

						SCREAMING_SNAKE_CASE_: Any  		=       str(tmp_path / "test_image_rgb.jpg"    )
						PIL.Image.fromarray(np.zeros((5, 5)				,	dtype=np.uinta    )    ).save(_UpperCAmelCase				,	format="png"    )
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferOutputStream()
						with ParquetWriter(
						    stream=_UpperCAmelCase				,	features=Features({"image": Image()}    )				,	embed_local_files=_UpperCAmelCase    ) as writer:
												writer.write({"image": image_path}    )
												writer.finalize()
						SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       pq.read_table(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: int  		=       pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out["image"][0]["path"]				,	_UpperCAmelCase    )
												with open(_UpperCAmelCase				,	"rb"    ) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: Dict  		=       pa.schema([pa.field("col_1"				,	pa.string()				,	nullable=_UpperCAmelCase    )]    )
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferOutputStream()
						with ArrowWriter(stream=_UpperCAmelCase    ) as writer:
												writer._build_writer(inferred_schema=_UpperCAmelCase    )
												assert writer._schema == pa.schema([pa.field("col_1"				,	pa.string()    )]    )









import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_       		=  get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class  A						(							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							AlbertTokenizer
    lowerCamelCase			=							AlbertTokenizerFast
    lowerCamelCase			=							True
    lowerCamelCase			=							True
    lowerCamelCase			=							True


    def 						snake_case__     (      self :       Dict		)->    Any:





        '''simple docstring'''







        super().setUp()

        # We have a SentencePiece fixture for testing
        A__          =   AlbertTokenizer(lowercase_		)
        tokenizer.save_pretrained(self.tmpdirname		)


    def 						snake_case__     (      self :       List[str],lowercase_ :       str		)->    Any:





        '''simple docstring'''







        A__          =   'this is a test'
        A__          =   'this is a test'
        return input_text, output_text


    def 						snake_case__     (      self :       List[Any]		)->    Optional[int]:





        '''simple docstring'''







        A__          =   '<pad>'
        A__          =   0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_		),lowercase_		)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_		),lowercase_		)


    def 						snake_case__     (      self :       List[str]		)->    str:





        '''simple docstring'''







        A__          =   list(self.get_tokenizer().get_vocab().keys()		)

        self.assertEqual(vocab_keys[0],'<pad>'		)
        self.assertEqual(vocab_keys[1],'<unk>'		)
        self.assertEqual(vocab_keys[-1],'▁eloquent'		)
        self.assertEqual(len(lowercase_		),3_0_0_0_0		)


    def 						snake_case__     (      self :       int		)->    List[Any]:





        '''simple docstring'''







        self.assertEqual(self.get_tokenizer().vocab_size,3_0_0_0_0		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    List[Any]:





        '''simple docstring'''







        if not self.test_rust_tokenizer:
            return

        A__          =   self.get_tokenizer()
        A__          =   self.get_rust_tokenizer()

        A__          =   'I was born in 92000, and this is falsé.'

        A__          =   tokenizer.tokenize(lowercase_		)
        A__          =   rust_tokenizer.tokenize(lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)

        A__          =   tokenizer.encode(lowercase_,add_special_tokens=lowercase_		)
        A__          =   rust_tokenizer.encode(lowercase_,add_special_tokens=lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)

        A__          =   self.get_rust_tokenizer()
        A__          =   tokenizer.encode(lowercase_		)
        A__          =   rust_tokenizer.encode(lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)


    def 						snake_case__     (      self :       int		)->    int:





        '''simple docstring'''







        A__          =   AlbertTokenizer(lowercase_,keep_accents=lowercase_		)

        A__          =   tokenizer.tokenize('This is a test'		)
        self.assertListEqual(lowercase_,['▁this', '▁is', '▁a', '▁test']		)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_		),[4_8, 2_5, 2_1, 1_2_8_9]		)

        A__          =   tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
        self.assertListEqual(
            lowercase_,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']		)
        A__          =   tokenizer.convert_tokens_to_ids(lowercase_		)
        self.assertListEqual(lowercase_,[3_1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]		)

        A__          =   tokenizer.convert_ids_to_tokens(lowercase_		)
        self.assertListEqual(
            lowercase_,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.'],)


    def 						snake_case__     (      self :       Union[str, Any]		)->    str:





        '''simple docstring'''







        A__          =   AlbertTokenizer(lowercase_		)

        A__          =   tokenizer.encode('sequence builders'		)
        A__          =   tokenizer.encode('multi-sequence build'		)

        A__          =   tokenizer.build_inputs_with_special_tokens(lowercase_		)
        A__          =   tokenizer.build_inputs_with_special_tokens(lowercase_,lowercase_		)

        assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
        assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
            tokenizer.sep_token_id
        ]


    @slow
    def 						snake_case__     (      self :       Any		)->    Tuple:





        '''simple docstring'''







        A__          =   {'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1_9_7_0, 1_3, 5, 6_0_9_2, 1_6_7, 2_8, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 7_0_2_8, 1_2_0_5_1, 1_8, 1_7, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 3_5_1_5, 1_8_6_8_4, 8, 4_4_6_1, 6, 1_9_2_7, 2_9_7, 8, 1_2_0_6_0, 2_6_0_7, 1_8, 1_3, 5, 4_4_6_1, 1_5, 1_0_5_3_8, 3_8, 8, 1_3_5, 1_5, 8_2_2, 5_8, 1_5, 9_9_3, 1_0_3_6_3, 1_5, 1_4_6_0, 8_0_0_5, 4_4_6_1, 1_5, 9_9_3, 2_5_5, 2_3_2_8, 9, 9, 9, 6, 2_6, 1_1_1_2, 8_1_6, 3_2_6_0, 1_3, 5, 1_0_3, 2_3_7_7, 6, 1_7, 1_1_1_2, 8_1_6, 2_7_8_2, 1_3, 5, 1_0_3, 1_0_6_4_1, 6, 2_9, 8_4, 2_5_1_2, 2_4_3_0, 7_8_2, 1_8_6_8_4, 2_7_6_1, 1_9, 8_0_8, 2_4_3_0, 2_5_5_6, 1_7, 8_5_5, 1_4_8_0, 9_4_7_7, 4_0_9_1, 1_2_8, 1_1_7_1_2, 1_5, 7_1_0_3, 2_1_5_3, 6_7_3, 1_7, 2_4_8_8_3, 9_9_9_0, 9, 3], [2, 1_1_5_0_2, 2_5, 1_0_0_6, 2_0, 7_8_2, 8, 1_1_8_0_9, 8_5_5, 1_7_3_2, 1_9_3_9_3, 1_8_6_6_7, 3_7, 3_6_7, 2_1_0_1_8, 6_9, 1_8_5_4, 3_4, 1_1_8_6_0, 1_9_1_2_4, 2_7, 1_5_6, 2_2_5, 1_7, 1_9_3, 4_1_4_1, 1_9, 6_5, 9_1_2_4, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1_4, 2_2_3_1, 8_8_6, 2_3_8_5, 1_7_6_5_9, 8_4, 1_4, 1_6_7_9_2, 1_9_5_2, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=lowercase_,model_name='albert-base-v2',revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e',)

