





import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =pa.array(TypedSequence([1, 2, 3]))
   self.assertEqual(arr.type       , pa.intaa())

 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->str:



   '''simple docstring'''



   with self.assertRaises(UpperCAmelCase_):
     lowerCamelCase__:      str			  =pa.array(TypedSequence([1, 2, 3])       , type=pa.intaa())

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Dict:



   '''simple docstring'''



   with self.assertRaises(UpperCAmelCase_):
     lowerCamelCase__:      str			  =pa.array(TypedSequence([1, 2, 3]       , try_type=Value("bool")       , type=Value("int64")))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  =pa.array(TypedSequence([1, 2, 3]       , type=Value("int32")))
   self.assertEqual(arr.type       , pa.intaa())

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Dict:



   '''simple docstring'''



   with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
     lowerCamelCase__:      Tuple			  =pa.array(TypedSequence(["foo", "bar"]       , type=Value("int64")))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any])       ->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:      Any			  =pa.array(TypedSequence([1, 2, 3]       , try_type=Value("int32")))
   self.assertEqual(arr.type       , pa.intaa())

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Tuple:



   '''simple docstring'''



   lowerCamelCase__:      Any			  =pa.array(TypedSequence(["foo", "bar"]       , try_type=Value("int64")))
   self.assertEqual(arr.type       , pa.string())

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  =pa.array(TypedSequence([[[1, 2, 3]]]       , type=ArrayaD((1, 3)       , "int64")))
   self.assertEqual(arr.type       , ArrayaDExtensionType((1, 3)       , "int64"))

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->str:



   '''simple docstring'''



   with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
     lowerCamelCase__:      Tuple			  =pa.array(TypedSequence(["foo", "bar"]       , type=ArrayaD((1, 3)       , "int64")))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  =pa.array(TypedSequence([[[1, 2, 3]]]       , try_type=ArrayaD((1, 3)       , "int64")))
   self.assertEqual(arr.type       , ArrayaDExtensionType((1, 3)       , "int64"))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Tuple			  =pa.array(TypedSequence(["foo", "bar"]       , try_type=ArrayaD((1, 3)       , "int64")))
   self.assertEqual(arr.type       , pa.string())







 @require_pil
 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int])       ->List[str]:



   '''simple docstring'''



   import PIL.Image

   lowerCamelCase__:      Any			  =PIL.Image.fromarray(np.arange(10       , dtype=np.uinta).reshape(2       , 5))
   with patch(
       "datasets.arrow_writer.cast_to_python_objects"       , side_effect=UpperCAmelCase_) as mock_cast_to_python_objects:
     lowerCamelCase__:      Optional[int]			  =pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]       , type=Image()))
     lowerCamelCase__  ,					lowerCamelCase__:      Dict			  =mock_cast_to_python_objects.call_args_list[-1]
     self.assertIn("optimize_list_casting"       , UpperCAmelCase_)
     self.assertFalse(kwargs["optimize_list_casting"])





def      lowerCAmelCase_ (       __a	,    __a		)     ->		List[str]:







  """simple docstring"""





  lowerCamelCase__:      Dict			  =pa.BufferReader(__a		) if isinstance(__a	,    pa.Buffer		) else pa.memory_map(__a		)
  lowerCamelCase__:      Union[str, Any]			  =pa.ipc.open_stream(__a		)
  lowerCamelCase__:      pa.Table			  =f.read_all()
  assert len(pa_table.to_batches()		) == expected_num_chunks
  assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
  del pa_table





@pytest.mark.parametrize("writer_batch_size"	,    [None, 1, 10]		)
@pytest.mark.parametrize(
    "fields"	,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Optional[Any]:







  """simple docstring"""





  lowerCamelCase__:      List[Any]			  =pa.BufferOutputStream()
  lowerCamelCase__:      List[Any]			  =pa.schema(__a		) if fields else None
  with ArrowWriter(stream=__a	,    schema=__a	,    writer_batch_size=__a		) as writer:
    writer.write({"col_1": "foo", "col_2": 1}		)
    writer.write({"col_1": "bar", "col_2": 2}		)
    lowerCamelCase__  ,					lowerCamelCase__:      int			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  if not fields:
    lowerCamelCase__:      Union[str, Any]			  ={"col_1": pa.string(), "col_2": pa.intaa()}
  assert writer._schema == pa.schema(__a	,    metadata=writer._schema.metadata		)
  _check_output(output.getvalue()	,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1		)





def      lowerCAmelCase_ (       )     ->		Tuple:







  """simple docstring"""





  lowerCamelCase__:      Union[str, Any]			  =pa.BufferOutputStream()
  lowerCamelCase__:      List[str]			  =Features({"labels": ClassLabel(names=["neg", "pos"]		)}		)
  with ArrowWriter(stream=__a	,    features=__a		) as writer:
    writer.write({"labels": 0}		)
    writer.write({"labels": 1}		)
    lowerCamelCase__  ,					lowerCamelCase__:      Any			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  assert writer._schema == features.arrow_schema
  assert writer._schema.metadata == features.arrow_schema.metadata
  lowerCamelCase__:      Union[str, Any]			  =pa.BufferReader(output.getvalue()		)
  lowerCamelCase__:      Dict			  =pa.ipc.open_stream(__a		)
  lowerCamelCase__:      pa.Table			  =f.read_all()
  lowerCamelCase__:      str			  =pa_table.schema
  assert pa_table.num_rows == 2
  assert schema == features.arrow_schema
  assert schema.metadata == features.arrow_schema.metadata
  assert features == Features.from_arrow_schema(__a		)





@pytest.mark.parametrize("writer_batch_size"	,    [None, 1, 10]		)
def      lowerCAmelCase_ (       __a		)     ->		int:







  """simple docstring"""





  lowerCamelCase__:      Tuple			  =pa.BufferOutputStream()
  with ArrowWriter(
      stream=__a	,    writer_batch_size=__a	,    hash_salt="split_name"	,    check_duplicates=__a	,    ) as writer:
    with pytest.raises(__a		):
      writer.write({"col_1": "foo", "col_2": 1}	,    key=[1, 2]		)
      lowerCamelCase__  ,					lowerCamelCase__:      Union[str, Any]			  =writer.finalize()





@pytest.mark.parametrize("writer_batch_size"	,    [None, 2, 10]		)
def      lowerCAmelCase_ (       __a		)     ->		Union[str, Any]:







  """simple docstring"""





  lowerCamelCase__:      Any			  =pa.BufferOutputStream()
  with ArrowWriter(
      stream=__a	,    writer_batch_size=__a	,    hash_salt="split_name"	,    check_duplicates=__a	,    ) as writer:
    with pytest.raises(__a		):
      writer.write({"col_1": "foo", "col_2": 1}	,    key=10		)
      writer.write({"col_1": "bar", "col_2": 2}	,    key=10		)
      lowerCamelCase__  ,					lowerCamelCase__:      int			  =writer.finalize()





@pytest.mark.parametrize("writer_batch_size"	,    [None, 2, 10]		)
def      lowerCAmelCase_ (       __a		)     ->		Dict:







  """simple docstring"""





  lowerCamelCase__:      List[Any]			  =pa.BufferOutputStream()
  with ArrowWriter(
      stream=__a	,    writer_batch_size=__a	,    hash_salt="split_name"	,    check_duplicates=__a	,    ) as writer:
    writer.write({"col_1": "foo", "col_2": 1}	,    key=1		)
    writer.write({"col_1": "bar", "col_2": 2}	,    key=2		)
    lowerCamelCase__  ,					lowerCamelCase__:      str			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  _check_output(output.getvalue()	,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1		)





@pytest.mark.parametrize("writer_batch_size"	,    [None, 1, 10]		)
@pytest.mark.parametrize(
    "fields"	,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Dict:







  """simple docstring"""





  lowerCamelCase__:      Tuple			  =pa.BufferOutputStream()
  lowerCamelCase__:      List[Any]			  =pa.schema(__a		) if fields else None
  with ArrowWriter(stream=__a	,    schema=__a	,    writer_batch_size=__a		) as writer:
    writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}		)
    writer.write_batch({"col_1": [], "col_2": []}		)
    lowerCamelCase__  ,					lowerCamelCase__:      List[str]			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  if not fields:
    lowerCamelCase__:      Optional[int]			  ={"col_1": pa.string(), "col_2": pa.intaa()}
  assert writer._schema == pa.schema(__a	,    metadata=writer._schema.metadata		)
  _check_output(output.getvalue()	,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1		)





@pytest.mark.parametrize("writer_batch_size"	,    [None, 1, 10]		)
@pytest.mark.parametrize(
    "fields"	,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Union[str, Any]:







  """simple docstring"""





  lowerCamelCase__:      Dict			  =pa.BufferOutputStream()
  lowerCamelCase__:      Union[str, Any]			  =pa.schema(__a		) if fields else None
  with ArrowWriter(stream=__a	,    schema=__a	,    writer_batch_size=__a		) as writer:
    writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}		)		)
    lowerCamelCase__  ,					lowerCamelCase__:      Any			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  if not fields:
    lowerCamelCase__:      List[str]			  ={"col_1": pa.string(), "col_2": pa.intaa()}
  assert writer._schema == pa.schema(__a	,    metadata=writer._schema.metadata		)
  _check_output(output.getvalue()	,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1		)





@pytest.mark.parametrize("writer_batch_size"	,    [None, 1, 10]		)
@pytest.mark.parametrize(
    "fields"	,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		str:







  """simple docstring"""





  lowerCamelCase__:      Tuple			  =pa.BufferOutputStream()
  lowerCamelCase__:      Any			  =pa.schema(__a		) if fields else None
  with ArrowWriter(stream=__a	,    schema=__a	,    writer_batch_size=__a		) as writer:
    writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}		)		)
    writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}		)		)
    lowerCamelCase__  ,					lowerCamelCase__:      Union[str, Any]			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  if not fields:
    lowerCamelCase__:      int			  ={"col_1": pa.string(), "col_2": pa.intaa()}
  assert writer._schema == pa.schema(__a	,    metadata=writer._schema.metadata		)
  _check_output(output.getvalue()	,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1		)





def      lowerCAmelCase_ (       )     ->		Union[str, Any]:







  """simple docstring"""





  with tempfile.TemporaryDirectory() as tmp_dir:
    lowerCamelCase__:      List[str]			  ={"col_1": pa.string(), "col_2": pa.intaa()}
    lowerCamelCase__:      Tuple			  =os.path.join(__a	,    "test.arrow"		)
    with ArrowWriter(path=__a	,    schema=pa.schema(__a		)		) as writer:
      writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}		)
      lowerCamelCase__  ,					lowerCamelCase__:      Optional[int]			  =writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert writer._schema == pa.schema(__a	,    metadata=writer._schema.metadata		)
    _check_output(__a	,    1		)





def      lowerCAmelCase_ (       __a		)     ->		Tuple:







  """simple docstring"""





  if pa.types.is_list(__a		):
    return get_base_dtype(arr_type.value_type		)
  else:
    return arr_type





def      lowerCAmelCase_ (       __a	,    __a		)     ->		List[str]:







  """simple docstring"""





  if isinstance(lst[0]	,    __a		):
    change_first_primitive_element_in_list(lst[0]	,    __a		)
  else:
    lowerCamelCase__:      Tuple			  =value





@pytest.mark.parametrize("optimized_int_type, expected_dtype"	,    [(None, pa.intaa()), (Value("int32"		), pa.intaa())]		)
@pytest.mark.parametrize("sequence"	,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		str:







  """simple docstring"""





  lowerCamelCase__:      List[Any]			  =pa.array(TypedSequence(__a	,    optimized_int_type=__a		)		)
  assert get_base_dtype(arr.type		) == expected_dtype





@pytest.mark.parametrize(
    "col, expected_dtype"	,    [
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]	,    )
@pytest.mark.parametrize("sequence"	,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]		)
def      lowerCAmelCase_ (       __a	,    __a	,    __a		)     ->		Tuple:







  """simple docstring"""





  lowerCamelCase__:      Optional[Any]			  =pa.array(OptimizedTypedSequence(__a	,    col=__a		)		)
  assert get_base_dtype(arr.type		) == expected_dtype

  # not in range
  if col != "other":
    # avoids errors due to in-place modifications
    lowerCamelCase__:      Any			  =copy.deepcopy(__a		)
    lowerCamelCase__:      Any			  =np.iinfo(expected_dtype.to_pandas_dtype()		).max + 1
    change_first_primitive_element_in_list(__a	,    __a		)
    lowerCamelCase__:      Optional[Any]			  =pa.array(OptimizedTypedSequence(__a	,    col=__a		)		)
    assert get_base_dtype(arr.type		) == pa.intaa()





@pytest.mark.parametrize("raise_exception"	,    [False, True]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Tuple:







  """simple docstring"""





  lowerCamelCase__:      str			  =str(tmp_path / "dataset-train.arrow"		)
  try:
    with ArrowWriter(path=__a		) as writer:
      if raise_exception:
        raise pa.lib.ArrowInvalid()
      else:
        writer.stream.close()
  except pa.lib.ArrowInvalid:
    pass
  finally:
    assert writer.stream.closed





def      lowerCAmelCase_ (       __a		)     ->		List[str]:







  """simple docstring"""





  lowerCamelCase__:      str			  ="mock://dataset-train.arrow"
  with ArrowWriter(path=__a	,    storage_options=mockfs.storage_options		) as writer:
    assert isinstance(writer._fs	,    type(__a		)		)
    assert writer._fs.storage_options == mockfs.storage_options
    writer.write({"col_1": "foo", "col_2": 1}		)
    writer.write({"col_1": "bar", "col_2": 2}		)
    lowerCamelCase__  ,					lowerCamelCase__:      str			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  assert mockfs.exists(__a		)





def      lowerCAmelCase_ (       )     ->		Any:







  """simple docstring"""





  lowerCamelCase__:      Union[str, Any]			  =pa.BufferOutputStream()
  with ParquetWriter(stream=__a		) as writer:
    writer.write({"col_1": "foo", "col_2": 1}		)
    writer.write({"col_1": "bar", "col_2": 2}		)
    lowerCamelCase__  ,					lowerCamelCase__:      Optional[Any]			  =writer.finalize()
  assert num_examples == 2
  assert num_bytes > 0
  lowerCamelCase__:      Tuple			  =pa.BufferReader(output.getvalue()		)
  lowerCamelCase__:      pa.Table			  =pq.read_table(__a		)
  assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}





@require_pil
@pytest.mark.parametrize("embed_local_files"	,    [False, True]		)
def      lowerCAmelCase_ (       __a	,    __a		)     ->		Any:







  """simple docstring"""





  import PIL.Image

  lowerCamelCase__:      Tuple			  =str(tmp_path / "test_image_rgb.jpg"		)
  PIL.Image.fromarray(np.zeros((5, 5)	,    dtype=np.uinta		)		).save(__a	,    format="png"		)
  lowerCamelCase__:      Union[str, Any]			  =pa.BufferOutputStream()
  with ParquetWriter(
      stream=__a	,    features=Features({"image": Image()}		)	,    embed_local_files=__a		) as writer:
    writer.write({"image": image_path}		)
    writer.finalize()
  lowerCamelCase__:      Any			  =pa.BufferReader(output.getvalue()		)
  lowerCamelCase__:      pa.Table			  =pq.read_table(__a		)
  lowerCamelCase__:      Tuple			  =pa_table.to_pydict()
  if embed_local_files:
    assert isinstance(out["image"][0]["path"]	,    __a		)
    with open(__a	,    "rb"		) as f:
      assert out["image"][0]["bytes"] == f.read()
  else:
    assert out["image"][0]["path"] == image_path
    assert out["image"][0]["bytes"] is None






def      lowerCAmelCase_ (       )     ->		Optional[int]:







  """simple docstring"""





  lowerCamelCase__:      Union[str, Any]			  =pa.schema([pa.field("col_1"	,    pa.string()	,    nullable=__a		)]		)
  lowerCamelCase__:      Dict			  =pa.BufferOutputStream()
  with ArrowWriter(stream=__a		) as writer:
    writer._build_writer(inferred_schema=__a		)
    assert writer._schema == pa.schema([pa.field("col_1"	,    pa.string()		)]		)








import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


__A									=				get_tests_dir("fixtures/test_sentencepiece_bpe.model")



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			BartphoTokenizer
 lowercase_		=			False
 lowercase_		=			True

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Tuple:



   '''simple docstring'''



   super().setUp()

   lowerCamelCase__:      int			  =["▁This", "▁is", "▁a", "▁t", "est"]
   lowerCamelCase__:      Tuple			  =dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_))))
   lowerCamelCase__:      List[Any]			  ={"unk_token": "<unk>"}

   lowerCamelCase__:      Dict			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["monolingual_vocab_file"])
   with open(self.monolingual_vocab_file       , "w"       , encoding="utf-8") as fp:
     for token in vocab_tokens:
       fp.write(F"""{token} {vocab_tokens[token]}\n""")

   lowerCamelCase__:      Dict			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   tokenizer.save_pretrained(self.tmpdirname)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int]       , **UpperCAmelCase_				:   Optional[Any])       ->str:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return BartphoTokenizer.from_pretrained(self.tmpdirname       , **UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any]       , UpperCAmelCase_				:   Optional[Any])       ->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  ="This is a là test"
   lowerCamelCase__:      Optional[Any]			  ="This is a<unk><unk> test"
   return input_text, output_text







 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      str			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   lowerCamelCase__:      List[Any]			  ="This is a là test"
   lowerCamelCase__:      Optional[int]			  ="▁This ▁is ▁a ▁l à ▁t est".split()
   lowerCamelCase__:      Optional[int]			  =tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Tuple			  =tokens + [tokenizer.unk_token]
   lowerCamelCase__:      List[Any]			  =[4, 5, 6, 3, 3, 7, 8, 3]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , UpperCAmelCase_)

