"""simple docstring"""







import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Union[str, Any]						)    ->     Optional[int]:







     '''simple docstring'''


     __A					:					Tuple								=		tmp_path / 'file.jsonl'
     __A					:					int								=		textwrap.dedent(
         '\\n        {"col_1": -1}\n        {"col_1": 1, "col_2": 2}\n        {"col_1": 10, "col_2": 20}\n        '						)
     with open(_SCREAMING_SNAKE_CASE  ,       'w'						) as f:
          f.write(_SCREAMING_SNAKE_CASE						)
     return str(_SCREAMING_SNAKE_CASE						)




@pytest.fixture
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Optional[int]						)    ->     Optional[Any]:







     '''simple docstring'''


     __A					:					List[Any]								=		tmp_path / 'file_utf16_encoded.jsonl'
     __A					:					Optional[int]								=		textwrap.dedent(
         '\\n        {"col_1": -1}\n        {"col_1": 1, "col_2": 2}\n        {"col_1": 10, "col_2": 20}\n        '						)
     with open(_SCREAMING_SNAKE_CASE  ,       'w'  ,       encoding='utf-16'						) as f:
          f.write(_SCREAMING_SNAKE_CASE						)
     return str(_SCREAMING_SNAKE_CASE						)




@pytest.fixture
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Tuple						)    ->     Optional[int]:







     '''simple docstring'''


     __A					:					Optional[Any]								=		tmp_path / 'file_with_list_of_dicts.json'
     __A					:					List[str]								=		textwrap.dedent(
         '\\n        [\n            {"col_1": -1},\n            {"col_1": 1, "col_2": 2},\n            {"col_1": 10, "col_2": 20}\n        ]\n        '						)
     with open(_SCREAMING_SNAKE_CASE  ,       'w'						) as f:
          f.write(_SCREAMING_SNAKE_CASE						)
     return str(_SCREAMING_SNAKE_CASE						)




@pytest.fixture
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Optional[int]						)    ->     str:







     '''simple docstring'''


     __A					:					Any								=		tmp_path / 'file_with_list_of_dicts_field.json'
     __A					:					Tuple								=		textwrap.dedent(
         '\\n        {\n            "field1": 1,\n            "field2": "aabb",\n            "field3": [\n                {"col_1": -1},\n                {"col_1": 1, "col_2": 2},\n                {"col_1": 10, "col_2": 20}\n            ]\n        }\n        '						)
     with open(_SCREAMING_SNAKE_CASE  ,       'w'						) as f:
          f.write(_SCREAMING_SNAKE_CASE						)
     return str(_SCREAMING_SNAKE_CASE						)




@pytest.mark.parametrize(
    'file_fixture, config_kwargs'  ,       [
        ('jsonl_file', {}),
        ('jsonl_file_utf16_encoded', {'encoding': 'utf-16'}),
        ('json_file_with_list_of_dicts', {}),
        ('json_file_with_list_of_dicts_field', {'field': 'field3'}),
    ]  ,       )
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Dict  ,       _SCREAMING_SNAKE_CASE				:				Optional[int]  ,       _SCREAMING_SNAKE_CASE				:				Tuple						)    ->     Optional[int]:







     '''simple docstring'''


     __A					:					Dict								=		Json(**_SCREAMING_SNAKE_CASE						)
     __A					:					int								=		json._generate_tables([[request.getfixturevalue(_SCREAMING_SNAKE_CASE						)]]						)
     __A					:					List[Any]								=		pa.concat_tables([table for _, table in generator]						)
     assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    'file_fixture, config_kwargs'  ,       [
        (
            'jsonl_file',
            {'features': Features({'col_1': Value('int64'						), 'col_2': Value('int64'						), 'missing_col': Value('string'						)}						)},
        ),
        (
            'json_file_with_list_of_dicts',
            {'features': Features({'col_1': Value('int64'						), 'col_2': Value('int64'						), 'missing_col': Value('string'						)}						)},
        ),
        (
            'json_file_with_list_of_dicts_field',
            {
                'field': 'field3',
                'features': Features(
                    {'col_1': Value('int64'						), 'col_2': Value('int64'						), 'missing_col': Value('string'						)}						),
            },
        ),
    ]  ,       )
def 						_lowercase				(  _SCREAMING_SNAKE_CASE				:				Tuple  ,       _SCREAMING_SNAKE_CASE				:				Optional[Any]  ,       _SCREAMING_SNAKE_CASE				:				List[str]						)    ->     str:







     '''simple docstring'''


     __A					:					Optional[Any]								=		Json(**_SCREAMING_SNAKE_CASE						)
     __A					:					List[str]								=		json._generate_tables([[request.getfixturevalue(_SCREAMING_SNAKE_CASE						)]]						)
     __A					:					Optional[int]								=		pa.concat_tables([table for _, table in generator]						)
     assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}




"""simple docstring"""







import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp




def 						_lowercase				(  )    ->     List[str]:







     '''simple docstring'''


     __A					:					int								=		ArgumentParser(
         description=(
             'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
         )						)

     # Optional arguments for the launch helper
     parser.add_argument('--num_cores'  ,       type=_SCREAMING_SNAKE_CASE  ,       default=1  ,       help='Number of TPU cores to use (1 or 8).'						)

     # positional
     parser.add_argument(
         'training_script'  ,       type=_SCREAMING_SNAKE_CASE  ,       help=(
             'The full path to the single TPU training '
             'program/script to be launched in parallel, '
             'followed by all the arguments for the '
             'training script'
         )  ,       )

     # rest from the training program
     parser.add_argument('training_script_args'  ,       nargs=_SCREAMING_SNAKE_CASE						)

     return parser.parse_args()




def 						_lowercase				(  )    ->     List[str]:







     '''simple docstring'''


     __A					:					List[str]								=		parse_args()

     # Import training_script as a module.
     __A					:					Dict								=		Path(args.training_script						)
     sys.path.append(str(script_fpath.parent.resolve()						)						)
     __A					:					List[Any]								=		script_fpath.stem
     __A					:					Optional[Any]								=		importlib.import_module(_SCREAMING_SNAKE_CASE						)

     # Patch sys.argv
     __A					:					Union[str, Any]								=		[args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores						)]

     xmp.spawn(mod._mp_fn  ,       args=()  ,       nprocs=args.num_cores						)


if __name__ == "__main__":
       main()



