

import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil

class    _a (       lowerCAmelCase_							):




      """simple docstring"""

      def 						__A     (					self				:							List[Any]     ):
            A_		   =							pa.array(TypedSequence([1, 2, 3]     )     )
            self.assertEqual(arr.type		,       pa.intaa()     )





      def 						__A     (					self				:							Optional[Any]     ):
            with self.assertRaises(snake_case_     ):
                  A_		   =							pa.array(TypedSequence([1, 2, 3]     )		,       type=pa.intaa()     )





      def 						__A     (					self				:							Dict     ):
            with self.assertRaises(snake_case_     ):
                  A_		   =							pa.array(TypedSequence([1, 2, 3]		,       try_type=Value("bool"     )		,       type=Value("int64"     )     )     )





      def 						__A     (					self				:							str     ):
            A_		   =							pa.array(TypedSequence([1, 2, 3]		,       type=Value("int32"     )     )     )
            self.assertEqual(arr.type		,       pa.intaa()     )





      def 						__A     (					self				:							Union[str, Any]     ):
            with self.assertRaises((TypeError, pa.lib.ArrowInvalid)     ):
                  A_		   =							pa.array(TypedSequence(["foo", "bar"]		,       type=Value("int64"     )     )     )





      def 						__A     (					self				:							int     ):
            A_		   =							pa.array(TypedSequence([1, 2, 3]		,       try_type=Value("int32"     )     )     )
            self.assertEqual(arr.type		,       pa.intaa()     )





      def 						__A     (					self				:							Dict     ):
            A_		   =							pa.array(TypedSequence(["foo", "bar"]		,       try_type=Value("int64"     )     )     )
            self.assertEqual(arr.type		,       pa.string()     )





      def 						__A     (					self				:							int     ):
            A_		   =							pa.array(TypedSequence([[[1, 2, 3]]]		,       type=ArrayaD((1, 3)		,       "int64"     )     )     )
            self.assertEqual(arr.type		,       ArrayaDExtensionType((1, 3)		,       "int64"     )     )





      def 						__A     (					self				:							Dict     ):
            with self.assertRaises((TypeError, pa.lib.ArrowInvalid)     ):
                  A_		   =							pa.array(TypedSequence(["foo", "bar"]		,       type=ArrayaD((1, 3)		,       "int64"     )     )     )





      def 						__A     (					self				:							str     ):
            A_		   =							pa.array(TypedSequence([[[1, 2, 3]]]		,       try_type=ArrayaD((1, 3)		,       "int64"     )     )     )
            self.assertEqual(arr.type		,       ArrayaDExtensionType((1, 3)		,       "int64"     )     )





      def 						__A     (					self				:							Tuple     ):
            A_		   =							pa.array(TypedSequence(["foo", "bar"]		,       try_type=ArrayaD((1, 3)		,       "int64"     )     )     )
            self.assertEqual(arr.type		,       pa.string()     )







      @require_pil
      def 						__A     (					self				:							Optional[int]     ):
            import PIL.Image

            A_		   =							PIL.Image.fromarray(np.arange(10		,       dtype=np.uinta     ).reshape(2		,       5     )     )
            with patch(
                "datasets.arrow_writer.cast_to_python_objects"		,       side_effect=snake_case_     ) as mock_cast_to_python_objects:
                  A_		   =							pa.array(TypedSequence([{"path": None, "bytes": B"image_bytes"}, pil_image]		,       type=Image()     )     )
                  A_   ,			A_		   =							mock_cast_to_python_objects.call_args_list[-1]
                  self.assertIn("optimize_list_casting"		,       snake_case_     )
                  self.assertFalse(kwargs["optimize_list_casting"]     )




def 	__snake_case		(      __UpperCamelCase     :      List[str]							,__UpperCamelCase     :      int  ):



      """simple docstring"""

      A_		   =							pa.BufferReader(__lowercase  ) if isinstance(__lowercase							,pa.Buffer  ) else pa.memory_map(__lowercase  )
      A_		   =							pa.ipc.open_stream(__lowercase  )
      A_		   =							f.read_all()
      assert len(pa_table.to_batches()  ) == expected_num_chunks
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
      del pa_table

@pytest.mark.parametrize("writer_batch_size"							,[None, 1, 10]  )
@pytest.mark.parametrize(
    "fields"							,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]  )
def 	__snake_case		(      __UpperCamelCase     :      Tuple							,__UpperCamelCase     :      int  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      A_		   =							pa.schema(__lowercase  ) if fields else None
      with ArrowWriter(stream=__lowercase							,schema=__lowercase							,writer_batch_size=__lowercase  ) as writer:
            writer.write({"col_1": "foo", "col_2": 1}  )
            writer.write({"col_1": "bar", "col_2": 2}  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            A_		   =							{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(__lowercase							,metadata=writer._schema.metadata  )
      _check_output(output.getvalue()							,expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )

def 	__snake_case		(      ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      A_		   =							Features({"labels": ClassLabel(names=["neg", "pos"]  )}  )
      with ArrowWriter(stream=__lowercase							,features=__lowercase  ) as writer:
            writer.write({"labels": 0}  )
            writer.write({"labels": 1}  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert writer._schema == features.arrow_schema
      assert writer._schema.metadata == features.arrow_schema.metadata
      A_		   =							pa.BufferReader(output.getvalue()  )
      A_		   =							pa.ipc.open_stream(__lowercase  )
      A_		   =							f.read_all()
      A_		   =							pa_table.schema
      assert pa_table.num_rows == 2
      assert schema == features.arrow_schema
      assert schema.metadata == features.arrow_schema.metadata
      assert features == Features.from_arrow_schema(__lowercase  )

@pytest.mark.parametrize("writer_batch_size"							,[None, 1, 10]  )
def 	__snake_case		(      __UpperCamelCase     :      List[Any]  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      with ArrowWriter(
          stream=__lowercase							,writer_batch_size=__lowercase							,hash_salt="split_name"							,check_duplicates=__lowercase							,) as writer:
            with pytest.raises(__lowercase  ):
                  writer.write({"col_1": "foo", "col_2": 1}							,key=[1, 2]  )
                  A_   ,			A_		   =							writer.finalize()

@pytest.mark.parametrize("writer_batch_size"							,[None, 2, 10]  )
def 	__snake_case		(      __UpperCamelCase     :      Optional[Any]  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      with ArrowWriter(
          stream=__lowercase							,writer_batch_size=__lowercase							,hash_salt="split_name"							,check_duplicates=__lowercase							,) as writer:
            with pytest.raises(__lowercase  ):
                  writer.write({"col_1": "foo", "col_2": 1}							,key=10  )
                  writer.write({"col_1": "bar", "col_2": 2}							,key=10  )
                  A_   ,			A_		   =							writer.finalize()

@pytest.mark.parametrize("writer_batch_size"							,[None, 2, 10]  )
def 	__snake_case		(      __UpperCamelCase     :      Union[str, Any]  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      with ArrowWriter(
          stream=__lowercase							,writer_batch_size=__lowercase							,hash_salt="split_name"							,check_duplicates=__lowercase							,) as writer:
            writer.write({"col_1": "foo", "col_2": 1}							,key=1  )
            writer.write({"col_1": "bar", "col_2": 2}							,key=2  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      _check_output(output.getvalue()							,expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )

@pytest.mark.parametrize("writer_batch_size"							,[None, 1, 10]  )
@pytest.mark.parametrize(
    "fields"							,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]  )
def 	__snake_case		(      __UpperCamelCase     :      str							,__UpperCamelCase     :      Optional[int]  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      A_		   =							pa.schema(__lowercase  ) if fields else None
      with ArrowWriter(stream=__lowercase							,schema=__lowercase							,writer_batch_size=__lowercase  ) as writer:
            writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}  )
            writer.write_batch({"col_1": [], "col_2": []}  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            A_		   =							{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(__lowercase							,metadata=writer._schema.metadata  )
      _check_output(output.getvalue()							,expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )

@pytest.mark.parametrize("writer_batch_size"							,[None, 1, 10]  )
@pytest.mark.parametrize(
    "fields"							,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]  )
def 	__snake_case		(      __UpperCamelCase     :      str							,__UpperCamelCase     :      Any  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      A_		   =							pa.schema(__lowercase  ) if fields else None
      with ArrowWriter(stream=__lowercase							,schema=__lowercase							,writer_batch_size=__lowercase  ) as writer:
            writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}  )  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            A_		   =							{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(__lowercase							,metadata=writer._schema.metadata  )
      _check_output(output.getvalue()							,expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )

@pytest.mark.parametrize("writer_batch_size"							,[None, 1, 10]  )
@pytest.mark.parametrize(
    "fields"							,[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]  )
def 	__snake_case		(      __UpperCamelCase     :      Union[str, Any]							,__UpperCamelCase     :      Tuple  ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      A_		   =							pa.schema(__lowercase  ) if fields else None
      with ArrowWriter(stream=__lowercase							,schema=__lowercase							,writer_batch_size=__lowercase  ) as writer:
            writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}  )  )
            writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}  )  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            A_		   =							{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(__lowercase							,metadata=writer._schema.metadata  )
      _check_output(output.getvalue()							,expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )

def 	__snake_case		(      ):



      """simple docstring"""

      with tempfile.TemporaryDirectory() as tmp_dir:
            A_		   =							{"col_1": pa.string(), "col_2": pa.intaa()}
            A_		   =							os.path.join(__lowercase							,"test.arrow"  )
            with ArrowWriter(path=__lowercase							,schema=pa.schema(__lowercase  )  ) as writer:
                  writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}  )
                  A_   ,			A_		   =							writer.finalize()
            assert num_examples == 2
            assert num_bytes > 0
            assert writer._schema == pa.schema(__lowercase							,metadata=writer._schema.metadata  )
            _check_output(__lowercase							,1  )

def 	__snake_case		(      __UpperCamelCase     :      str  ):



      """simple docstring"""

      if pa.types.is_list(__lowercase  ):
            return get_base_dtype(arr_type.value_type  )
      else:
            return arr_type

def 	__snake_case		(      __UpperCamelCase     :      List[Any]							,__UpperCamelCase     :      Optional[Any]  ):



      """simple docstring"""

      if isinstance(lst[0]							,__lowercase  ):
            change_first_primitive_element_in_list(lst[0]							,__lowercase  )
      else:
            A_		   =							value

@pytest.mark.parametrize("optimized_int_type, expected_dtype"							,[(None, pa.intaa()), (Value("int32"  ), pa.intaa())]  )
@pytest.mark.parametrize("sequence"							,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def 	__snake_case		(      __UpperCamelCase     :      Union[str, Any]							,__UpperCamelCase     :      str							,__UpperCamelCase     :      Dict  ):



      """simple docstring"""

      A_		   =							pa.array(TypedSequence(__lowercase							,optimized_int_type=__lowercase  )  )
      assert get_base_dtype(arr.type  ) == expected_dtype

@pytest.mark.parametrize(
    "col, expected_dtype"							,[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]							,)
@pytest.mark.parametrize("sequence"							,[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def 	__snake_case		(      __UpperCamelCase     :      Dict							,__UpperCamelCase     :      List[Any]							,__UpperCamelCase     :      List[Any]  ):



      """simple docstring"""

      A_		   =							pa.array(OptimizedTypedSequence(__lowercase							,col=__lowercase  )  )
      assert get_base_dtype(arr.type  ) == expected_dtype

      # not in range
      if col != "other":
            # avoids errors due to in-place modifications
            A_		   =							copy.deepcopy(__lowercase  )
            A_		   =							np.iinfo(expected_dtype.to_pandas_dtype()  ).max + 1
            change_first_primitive_element_in_list(__lowercase							,__lowercase  )
            A_		   =							pa.array(OptimizedTypedSequence(__lowercase							,col=__lowercase  )  )
            assert get_base_dtype(arr.type  ) == pa.intaa()

@pytest.mark.parametrize("raise_exception"							,[False, True]  )
def 	__snake_case		(      __UpperCamelCase     :      List[Any]							,__UpperCamelCase     :      Tuple  ):



      """simple docstring"""

      A_		   =							str(tmp_path / "dataset-train.arrow"  )
      try:
            with ArrowWriter(path=__lowercase  ) as writer:
                  if raise_exception:
                        raise pa.lib.ArrowInvalid()
                  else:
                        writer.stream.close()
      except pa.lib.ArrowInvalid:
            pass
      finally:
            assert writer.stream.closed

def 	__snake_case		(      __UpperCamelCase     :      str  ):



      """simple docstring"""

      A_		   =							"mock://dataset-train.arrow"
      with ArrowWriter(path=__lowercase							,storage_options=mockfs.storage_options  ) as writer:
            assert isinstance(writer._fs							,type(__lowercase  )  )
            assert writer._fs.storage_options == mockfs.storage_options
            writer.write({"col_1": "foo", "col_2": 1}  )
            writer.write({"col_1": "bar", "col_2": 2}  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert mockfs.exists(__lowercase  )

def 	__snake_case		(      ):



      """simple docstring"""

      A_		   =							pa.BufferOutputStream()
      with ParquetWriter(stream=__lowercase  ) as writer:
            writer.write({"col_1": "foo", "col_2": 1}  )
            writer.write({"col_1": "bar", "col_2": 2}  )
            A_   ,			A_		   =							writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      A_		   =							pa.BufferReader(output.getvalue()  )
      A_		   =							pq.read_table(__lowercase  )
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}

@require_pil
@pytest.mark.parametrize("embed_local_files"							,[False, True]  )
def 	__snake_case		(      __UpperCamelCase     :      str							,__UpperCamelCase     :      str  ):



      """simple docstring"""

      import PIL.Image

      A_		   =							str(tmp_path / "test_image_rgb.jpg"  )
      PIL.Image.fromarray(np.zeros((5, 5)							,dtype=np.uinta  )  ).save(__lowercase							,format="png"  )
      A_		   =							pa.BufferOutputStream()
      with ParquetWriter(
          stream=__lowercase							,features=Features({"image": Image()}  )							,embed_local_files=__lowercase  ) as writer:
            writer.write({"image": image_path}  )
            writer.finalize()
      A_		   =							pa.BufferReader(output.getvalue()  )
      A_		   =							pq.read_table(__lowercase  )
      A_		   =							pa_table.to_pydict()
      if embed_local_files:
            assert isinstance(out["image"][0]["path"]							,__lowercase  )
            with open(__lowercase							,"rb"  ) as f:
                  assert out["image"][0]["bytes"] == f.read()
      else:
            assert out["image"][0]["path"] == image_path
            assert out["image"][0]["bytes"] is None



def 	__snake_case		(      ):



      """simple docstring"""

      A_		   =							pa.schema([pa.field("col_1"							,pa.string()							,nullable=__lowercase  )]  )
      A_		   =							pa.BufferOutputStream()
      with ArrowWriter(stream=__lowercase  ) as writer:
            writer._build_writer(inferred_schema=__lowercase  )
            assert writer._schema == pa.schema([pa.field("col_1"							,pa.string()  )]  )






'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  __SCREAMING_SNAKE_CASE     :List[Any]       =				None

__SCREAMING_SNAKE_CASE     :Union[str, Any]       =				logging.get_logger(__name__)
__SCREAMING_SNAKE_CASE     :List[str]       =				{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE     :List[Any]       =				{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

__SCREAMING_SNAKE_CASE     :Optional[Any]       =				{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

__SCREAMING_SNAKE_CASE     :Optional[int]       =				'''▁'''







class   A_	(  lowerCAmelCase_			):
 _lowerCamelCase  : Optional[int]           =							VOCAB_FILES_NAMES
 _lowerCamelCase  : Any           =							PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase  : Tuple           =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase  : int           =							AlbertTokenizer




 def __init__(					self       :    Optional[Any]	,			snake_case_       :    Optional[Any]=None	,			snake_case_       :    Optional[Any]=None	,			snake_case_       :    Optional[Any]=True	,			snake_case_       :    str=True	,			snake_case_       :    Tuple=False	,			snake_case_       :    List[Any]="[CLS]"	,			snake_case_       :    Union[str, Any]="[SEP]"	,			snake_case_       :    str="<unk>"	,			snake_case_       :    Union[str, Any]="[SEP]"	,			snake_case_       :    List[Any]="<pad>"	,			snake_case_       :    List[str]="[CLS]"	,			snake_case_       :    int="[MASK]"	,			**snake_case_       :    Any	,			):
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.
      _UpperCAmelCase     							=   (
          AddedToken(snake_case_	,			lstrip=snake_case_	,			rstrip=snake_case_	,			normalized=snake_case_    )
          if isinstance(snake_case_	,			snake_case_    )
          else mask_token
      )

      super().__init__(
          snake_case_	,			tokenizer_file=snake_case_	,			do_lower_case=snake_case_	,			remove_space=snake_case_	,			keep_accents=snake_case_	,			bos_token=snake_case_	,			eos_token=snake_case_	,			unk_token=snake_case_	,			sep_token=snake_case_	,			pad_token=snake_case_	,			cls_token=snake_case_	,			mask_token=snake_case_	,			**snake_case_	,			)

      _UpperCAmelCase     							=   do_lower_case
      _UpperCAmelCase     							=   remove_space
      _UpperCAmelCase     							=   keep_accents
      _UpperCAmelCase     							=   vocab_file
      _UpperCAmelCase     							=   False if not self.vocab_file else True




 def      lowercase	(					self       :    Union[str, Any]	,			snake_case_       :    List[int]	,			snake_case_       :    Optional[List[int]] = None    ):
      _UpperCAmelCase     							=   [self.sep_token_id]
      _UpperCAmelCase     							=   [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep




 def      lowercase	(					self       :    Dict	,			snake_case_       :    List[int]	,			snake_case_       :    Optional[List[int]] = None    ):
      _UpperCAmelCase     							=   [self.sep_token_id]
      _UpperCAmelCase     							=   [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




 def      lowercase	(					self       :    Optional[Any]	,			snake_case_       :    str	,			snake_case_       :    Optional[str] = None    ):
      if not self.can_save_slow_tokenizer:
           raise ValueError(
               "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
               "tokenizer."    )

      if not os.path.isdir(snake_case_    ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
           return
      _UpperCAmelCase     							=   os.path.join(
          snake_case_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

      if os.path.abspath(self.vocab_file    ) != os.path.abspath(snake_case_    ):
           copyfile(self.vocab_file	,			snake_case_    )

      return (out_vocab_file,)




