

import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    def 						snake_case__     (      self :       Optional[Any]		)->    List[Any]:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence([1, 2, 3]		)		)
        self.assertEqual(arr.type,pa.intaa()		)


    def 						snake_case__     (      self :       Tuple		)->    Optional[Any]:





        '''simple docstring'''







        with self.assertRaises(lowercase_		):
            A__          =   pa.array(TypedSequence([1, 2, 3]		),type=pa.intaa()		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    List[Any]:





        '''simple docstring'''







        with self.assertRaises(lowercase_		):
            A__          =   pa.array(TypedSequence([1, 2, 3],try_type=Value('bool'		),type=Value('int64'		)		)		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    Dict:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence([1, 2, 3],type=Value('int32'		)		)		)
        self.assertEqual(arr.type,pa.intaa()		)


    def 						snake_case__     (      self :       List[str]		)->    Dict:





        '''simple docstring'''







        with self.assertRaises((TypeError, pa.lib.ArrowInvalid)		):
            A__          =   pa.array(TypedSequence(['foo', 'bar'],type=Value('int64'		)		)		)


    def 						snake_case__     (      self :       Optional[int]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence([1, 2, 3],try_type=Value('int32'		)		)		)
        self.assertEqual(arr.type,pa.intaa()		)


    def 						snake_case__     (      self :       Tuple		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence(['foo', 'bar'],try_type=Value('int64'		)		)		)
        self.assertEqual(arr.type,pa.string()		)


    def 						snake_case__     (      self :       Optional[int]		)->    Tuple:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence([[[1, 2, 3]]],type=ArrayaD((1, 3),'int64'		)		)		)
        self.assertEqual(arr.type,ArrayaDExtensionType((1, 3),'int64'		)		)


    def 						snake_case__     (      self :       Optional[Any]		)->    Optional[int]:





        '''simple docstring'''







        with self.assertRaises((TypeError, pa.lib.ArrowInvalid)		):
            A__          =   pa.array(TypedSequence(['foo', 'bar'],type=ArrayaD((1, 3),'int64'		)		)		)


    def 						snake_case__     (      self :       Optional[int]		)->    List[str]:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence([[[1, 2, 3]]],try_type=ArrayaD((1, 3),'int64'		)		)		)
        self.assertEqual(arr.type,ArrayaDExtensionType((1, 3),'int64'		)		)


    def 						snake_case__     (      self :       Union[str, Any]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   pa.array(TypedSequence(['foo', 'bar'],try_type=ArrayaD((1, 3),'int64'		)		)		)
        self.assertEqual(arr.type,pa.string()		)


    @require_pil
    def 						snake_case__     (      self :       int		)->    Union[str, Any]:





        '''simple docstring'''







        import PIL.Image

        A__          =   PIL.Image.fromarray(np.arange(1_0,dtype=np.uinta		).reshape(2,5		)		)
        with patch(
            'datasets.arrow_writer.cast_to_python_objects',side_effect=lowercase_		) as mock_cast_to_python_objects:
            A__          =   pa.array(TypedSequence([{'path': None, 'bytes': B'image_bytes'}, pil_image],type=Image()		)		)
            A__      ,						A__          =   mock_cast_to_python_objects.call_args_list[-1]
            self.assertIn('optimize_list_casting',lowercase_		)
            self.assertFalse(kwargs['optimize_list_casting']		)




def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	List[str]    ,			SCREAMING_SNAKE_CASE__			:	int      )							->      Optional[int]:





    '''simple docstring'''






    A__          =   pa.BufferReader(SCREAMING_SNAKE_CASE__      ) if isinstance(SCREAMING_SNAKE_CASE__    ,			pa.Buffer      ) else pa.memory_map(SCREAMING_SNAKE_CASE__      )
    A__          =   pa.ipc.open_stream(SCREAMING_SNAKE_CASE__      )
    A__          =   f.read_all()
    assert len(pa_table.to_batches()      ) == expected_num_chunks
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
    del pa_table
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 1, 10]      )
@pytest.mark.parametrize(
    'fields'    ,			[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	List[Any]    ,			SCREAMING_SNAKE_CASE__			:	Tuple      )							->      List[Any]:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    A__          =   pa.schema(SCREAMING_SNAKE_CASE__      ) if fields else None
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__    ,			schema=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write({'col_1': 'foo', 'col_2': 1}      )
        writer.write({'col_1': 'bar', 'col_2': 2}      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        A__          =   {'col_1': pa.string(), 'col_2': pa.intaa()}
    assert writer._schema == pa.schema(SCREAMING_SNAKE_CASE__    ,			metadata=writer._schema.metadata      )
    _check_output(output.getvalue()    ,			expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
def 						_snake_case(    )							->      List[Any]:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    A__          =   Features({'labels': ClassLabel(names=['neg', 'pos']      )}      )
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__    ,			features=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write({'labels': 0}      )
        writer.write({'labels': 1}      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert writer._schema == features.arrow_schema
    assert writer._schema.metadata == features.arrow_schema.metadata
    A__          =   pa.BufferReader(output.getvalue()      )
    A__          =   pa.ipc.open_stream(SCREAMING_SNAKE_CASE__      )
    A__          =   f.read_all()
    A__          =   pa_table.schema
    assert pa_table.num_rows == 2
    assert schema == features.arrow_schema
    assert schema.metadata == features.arrow_schema.metadata
    assert features == Features.from_arrow_schema(SCREAMING_SNAKE_CASE__      )
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 1, 10]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	str      )							->      str:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    with ArrowWriter(
        stream=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__    ,			hash_salt='split_name'    ,			check_duplicates=SCREAMING_SNAKE_CASE__    ,			) as writer:
        with pytest.raises(SCREAMING_SNAKE_CASE__      ):
            writer.write({'col_1': 'foo', 'col_2': 1}    ,			key=[1, 2]      )
            A__      ,						A__          =   writer.finalize()
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 2, 10]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	str      )							->      Optional[Any]:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    with ArrowWriter(
        stream=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__    ,			hash_salt='split_name'    ,			check_duplicates=SCREAMING_SNAKE_CASE__    ,			) as writer:
        with pytest.raises(SCREAMING_SNAKE_CASE__      ):
            writer.write({'col_1': 'foo', 'col_2': 1}    ,			key=10      )
            writer.write({'col_1': 'bar', 'col_2': 2}    ,			key=10      )
            A__      ,						A__          =   writer.finalize()
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 2, 10]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple      )							->      Optional[int]:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    with ArrowWriter(
        stream=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__    ,			hash_salt='split_name'    ,			check_duplicates=SCREAMING_SNAKE_CASE__    ,			) as writer:
        writer.write({'col_1': 'foo', 'col_2': 1}    ,			key=1      )
        writer.write({'col_1': 'bar', 'col_2': 2}    ,			key=2      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    _check_output(output.getvalue()    ,			expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 1, 10]      )
@pytest.mark.parametrize(
    'fields'    ,			[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	str      )							->      Tuple:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    A__          =   pa.schema(SCREAMING_SNAKE_CASE__      ) if fields else None
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__    ,			schema=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )
        writer.write_batch({'col_1': [], 'col_2': []}      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        A__          =   {'col_1': pa.string(), 'col_2': pa.intaa()}
    assert writer._schema == pa.schema(SCREAMING_SNAKE_CASE__    ,			metadata=writer._schema.metadata      )
    _check_output(output.getvalue()    ,			expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 1, 10]      )
@pytest.mark.parametrize(
    'fields'    ,			[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Dict    ,			SCREAMING_SNAKE_CASE__			:	Union[str, Any]      )							->      Union[str, Any]:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    A__          =   pa.schema(SCREAMING_SNAKE_CASE__      ) if fields else None
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__    ,			schema=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        A__          =   {'col_1': pa.string(), 'col_2': pa.intaa()}
    assert writer._schema == pa.schema(SCREAMING_SNAKE_CASE__    ,			metadata=writer._schema.metadata      )
    _check_output(output.getvalue()    ,			expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'    ,			[None, 1, 10]      )
@pytest.mark.parametrize(
    'fields'    ,			[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	int    ,			SCREAMING_SNAKE_CASE__			:	Tuple      )							->      Tuple:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    A__          =   pa.schema(SCREAMING_SNAKE_CASE__      ) if fields else None
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__    ,			schema=SCREAMING_SNAKE_CASE__    ,			writer_batch_size=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}      )      )
        writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}      )      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        A__          =   {'col_1': pa.string(), 'col_2': pa.intaa()}
    assert writer._schema == pa.schema(SCREAMING_SNAKE_CASE__    ,			metadata=writer._schema.metadata      )
    _check_output(output.getvalue()    ,			expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
def 						_snake_case(    )							->      str:





    '''simple docstring'''






    with tempfile.TemporaryDirectory() as tmp_dir:
        A__          =   {'col_1': pa.string(), 'col_2': pa.intaa()}
        A__          =   os.path.join(SCREAMING_SNAKE_CASE__    ,			'test.arrow'      )
        with ArrowWriter(path=SCREAMING_SNAKE_CASE__    ,			schema=pa.schema(SCREAMING_SNAKE_CASE__      )      ) as writer:
            writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )
            A__      ,						A__          =   writer.finalize()
        assert num_examples == 2
        assert num_bytes > 0
        assert writer._schema == pa.schema(SCREAMING_SNAKE_CASE__    ,			metadata=writer._schema.metadata      )
        _check_output(SCREAMING_SNAKE_CASE__    ,			1      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Dict      )							->      Any:





    '''simple docstring'''






    if pa.types.is_list(SCREAMING_SNAKE_CASE__      ):
        return get_base_dtype(arr_type.value_type      )
    else:
        return arr_type
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple    ,			SCREAMING_SNAKE_CASE__			:	str      )							->      Dict:





    '''simple docstring'''






    if isinstance(lst[0]    ,			SCREAMING_SNAKE_CASE__      ):
        change_first_primitive_element_in_list(lst[0]    ,			SCREAMING_SNAKE_CASE__      )
    else:
        A__          =   value
@pytest.mark.parametrize('optimized_int_type, expected_dtype'    ,			[(None, pa.intaa()), (Value('int32'      ), pa.intaa())]      )
@pytest.mark.parametrize('sequence'    ,			[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	List[str]      )							->      Any:





    '''simple docstring'''






    A__          =   pa.array(TypedSequence(SCREAMING_SNAKE_CASE__    ,			optimized_int_type=SCREAMING_SNAKE_CASE__      )      )
    assert get_base_dtype(arr.type      ) == expected_dtype
@pytest.mark.parametrize(
    'col, expected_dtype'    ,			[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]    ,			)
@pytest.mark.parametrize('sequence'    ,			[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	str    ,			SCREAMING_SNAKE_CASE__			:	Union[str, Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]      )							->      Union[str, Any]:





    '''simple docstring'''






    A__          =   pa.array(OptimizedTypedSequence(SCREAMING_SNAKE_CASE__    ,			col=SCREAMING_SNAKE_CASE__      )      )
    assert get_base_dtype(arr.type      ) == expected_dtype

    # not in range
    if col != "other":
        # avoids errors due to in-place modifications
        A__          =   copy.deepcopy(SCREAMING_SNAKE_CASE__      )
        A__          =   np.iinfo(expected_dtype.to_pandas_dtype()      ).max + 1
        change_first_primitive_element_in_list(SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__      )
        A__          =   pa.array(OptimizedTypedSequence(SCREAMING_SNAKE_CASE__    ,			col=SCREAMING_SNAKE_CASE__      )      )
        assert get_base_dtype(arr.type      ) == pa.intaa()
@pytest.mark.parametrize('raise_exception'    ,			[False, True]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	Tuple      )							->      Tuple:





    '''simple docstring'''






    A__          =   str(tmp_path / 'dataset-train.arrow'      )
    try:
        with ArrowWriter(path=SCREAMING_SNAKE_CASE__      ) as writer:
            if raise_exception:
                raise pa.lib.ArrowInvalid()
            else:
                writer.stream.close()
    except pa.lib.ArrowInvalid:
        pass
    finally:
        assert writer.stream.closed
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Any      )							->      Dict:





    '''simple docstring'''






    A__          =   'mock://dataset-train.arrow'
    with ArrowWriter(path=SCREAMING_SNAKE_CASE__    ,			storage_options=mockfs.storage_options      ) as writer:
        assert isinstance(writer._fs    ,			type(SCREAMING_SNAKE_CASE__      )      )
        assert writer._fs.storage_options == mockfs.storage_options
        writer.write({'col_1': 'foo', 'col_2': 1}      )
        writer.write({'col_1': 'bar', 'col_2': 2}      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert mockfs.exists(SCREAMING_SNAKE_CASE__      )
def 						_snake_case(    )							->      int:





    '''simple docstring'''






    A__          =   pa.BufferOutputStream()
    with ParquetWriter(stream=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write({'col_1': 'foo', 'col_2': 1}      )
        writer.write({'col_1': 'bar', 'col_2': 2}      )
        A__      ,						A__          =   writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    A__          =   pa.BufferReader(output.getvalue()      )
    A__          =   pq.read_table(SCREAMING_SNAKE_CASE__      )
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize('embed_local_files'    ,			[False, True]      )
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	List[str]    ,			SCREAMING_SNAKE_CASE__			:	List[str]      )							->      List[Any]:





    '''simple docstring'''






    import PIL.Image

    A__          =   str(tmp_path / 'test_image_rgb.jpg'      )
    PIL.Image.fromarray(np.zeros((5, 5)    ,			dtype=np.uinta      )      ).save(SCREAMING_SNAKE_CASE__    ,			format='png'      )
    A__          =   pa.BufferOutputStream()
    with ParquetWriter(
        stream=SCREAMING_SNAKE_CASE__    ,			features=Features({'image': Image()}      )    ,			embed_local_files=SCREAMING_SNAKE_CASE__      ) as writer:
        writer.write({'image': image_path}      )
        writer.finalize()
    A__          =   pa.BufferReader(output.getvalue()      )
    A__          =   pq.read_table(SCREAMING_SNAKE_CASE__      )
    A__          =   pa_table.to_pydict()
    if embed_local_files:
        assert isinstance(out['image'][0]['path']    ,			SCREAMING_SNAKE_CASE__      )
        with open(SCREAMING_SNAKE_CASE__    ,			'rb'      ) as f:
            assert out["image"][0]["bytes"] == f.read()
    else:
        assert out["image"][0]["path"] == image_path
        assert out["image"][0]["bytes"] is None

def 						_snake_case(    )							->      Union[str, Any]:





    '''simple docstring'''






    A__          =   pa.schema([pa.field('col_1'    ,			pa.string()    ,			nullable=SCREAMING_SNAKE_CASE__      )]      )
    A__          =   pa.BufferOutputStream()
    with ArrowWriter(stream=SCREAMING_SNAKE_CASE__      ) as writer:
        writer._build_writer(inferred_schema=SCREAMING_SNAKE_CASE__      )
        assert writer._schema == pa.schema([pa.field('col_1'    ,			pa.string()      )]      )




from argparse import ArgumentParser

from datasets.commands.convert import ConvertCommand
from datasets.commands.dummy_data import DummyDataCommand
from datasets.commands.env import EnvironmentCommand
from datasets.commands.run_beam import RunBeamCommand
from datasets.commands.test import TestCommand
from datasets.utils.logging import set_verbosity_info
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple      )							->      Tuple:





    '''simple docstring'''






    return {key.lstrip('-'      ): value for key, value in zip(unknown_args[::2]    ,			unknown_args[1::2]      )}
def 						_snake_case(    )							->      Dict:





    '''simple docstring'''






    A__          =   ArgumentParser(
        'HuggingFace Datasets CLI tool'    ,			usage='datasets-cli <command> [<args>]'    ,			allow_abbrev=SCREAMING_SNAKE_CASE__      )
    A__          =   parser.add_subparsers(help='datasets-cli command helpers'      )
    set_verbosity_info()

    # Register commands
    ConvertCommand.register_subcommand(SCREAMING_SNAKE_CASE__      )
    EnvironmentCommand.register_subcommand(SCREAMING_SNAKE_CASE__      )
    TestCommand.register_subcommand(SCREAMING_SNAKE_CASE__      )
    RunBeamCommand.register_subcommand(SCREAMING_SNAKE_CASE__      )
    DummyDataCommand.register_subcommand(SCREAMING_SNAKE_CASE__      )

    # Parse args
    A__      ,						A__          =   parser.parse_known_args()
    if not hasattr(SCREAMING_SNAKE_CASE__    ,			'func'      ):
        parser.print_help()
        exit(1      )
    A__          =   parse_unknown_args(SCREAMING_SNAKE_CASE__      )

    # Run
    A__          =   args.func(SCREAMING_SNAKE_CASE__    ,			**SCREAMING_SNAKE_CASE__      )
    service.run()


if __name__ == "__main__":
     main()

