
"""simple docstring"""



import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


UpperCAmelCase__          = logging.get_logger(__name__)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Dict ,					__lowerCamelCase   :       List[str] ,					__lowerCamelCase   :       List[str] ,					__lowerCamelCase   :       List[str]=False		)     ->					List[str]:
  try:
    import torch  # noqa: F401
  except ImportError:
    logger.error(
        '''Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see'''
        ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
        ''' instructions.'''		)
    raise

  if not is_sharded:
    _snake_case					     =       os.path.abspath(_lowerCAmelCase		)
    logger.info(f'''Loading PyTorch weights from {pt_path}'''		)

    _snake_case					     =       torch.load(_lowerCAmelCase ,					map_location='''cpu'''		)
    logger.info(f'''PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()		):,} parameters.'''		)

    _snake_case					     =       convert_pytorch_state_dict_to_flax(_lowerCAmelCase ,					_lowerCAmelCase		)
  else:
    # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
    _snake_case					     =       convert_pytorch_sharded_state_dict_to_flax(_lowerCAmelCase ,					_lowerCAmelCase		)
  return flax_state_dict
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Tuple[str] ,					__lowerCamelCase   :       np.ndarray ,					__lowerCamelCase   :       Dict[str, jnp.ndarray] ,					__lowerCamelCase   :       str ,					)     ->					(Tuple[str], np.ndarray):

  def is_key_or_prefix_key_in_dict(__lowerCamelCase   :       Tuple[str]		) -> bool:
    return len(set(_lowerCAmelCase		) & {key, (model_prefix,) + key}		) > 0

  # layer norm
  _snake_case					     =       pt_tuple_key[:-1] + ('''scale''',)
  if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    return renamed_pt_tuple_key, pt_tensor

  # batch norm layer mean
  _snake_case					     =       pt_tuple_key[:-1] + ('''mean''',)
  if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    return renamed_pt_tuple_key, pt_tensor

  # batch norm layer var
  _snake_case					     =       pt_tuple_key[:-1] + ('''var''',)
  if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    return renamed_pt_tuple_key, pt_tensor

  # embedding
  _snake_case					     =       pt_tuple_key[:-1] + ('''embedding''',)
  if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    return renamed_pt_tuple_key, pt_tensor

  # conv layer
  _snake_case					     =       pt_tuple_key[:-1] + ('''kernel''',)
  if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    _snake_case					     =       pt_tensor.transpose(2 ,					3 ,					1 ,					0		)
    return renamed_pt_tuple_key, pt_tensor

  # linear layer
  _snake_case					     =       pt_tuple_key[:-1] + ('''kernel''',)
  if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
    _snake_case					     =       pt_tensor.T
    return renamed_pt_tuple_key, pt_tensor

  # old PyTorch layer norm weight
  _snake_case					     =       pt_tuple_key[:-1] + ('''weight''',)
  if pt_tuple_key[-1] == "gamma":
    return renamed_pt_tuple_key, pt_tensor

  # old PyTorch layer norm bias
  _snake_case					     =       pt_tuple_key[:-1] + ('''bias''',)
  if pt_tuple_key[-1] == "beta":
    return renamed_pt_tuple_key, pt_tensor

  # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
  _snake_case					     =       None
  if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
    _snake_case					     =       pt_tuple_key[-2] + '''_g'''
  elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
    _snake_case					     =       pt_tuple_key[-2] + '''_v'''
  if name is not None:
    _snake_case					     =       pt_tuple_key[:-3] + (name,)
    return renamed_pt_tuple_key, pt_tensor

  return pt_tuple_key, pt_tensor
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Optional[Any] ,					__lowerCamelCase   :       Optional[Any]		)     ->					Optional[Any]:
  # convert pytorch tensor to numpy
  _snake_case					     =       {k: v.numpy() for k, v in pt_state_dict.items()}

  _snake_case					     =       flax_model.base_model_prefix

  # use params dict if the model contains batch norm layers
  if "params" in flax_model.params:
    _snake_case					     =       flax_model.params['''params''']
  else:
    _snake_case					     =       flax_model.params
  _snake_case					     =       flatten_dict(_lowerCAmelCase		)

  # add batch_stats keys,values to dict
  if "batch_stats" in flax_model.params:
    _snake_case					     =       flatten_dict(flax_model.params['''batch_stats''']		)
    random_flax_state_dict.update(_lowerCAmelCase		)

  _snake_case					     =       {}

  _snake_case					     =       (model_prefix not in flax_model_params) and (
      model_prefix in {k.split('''.'''		)[0] for k in pt_state_dict.keys()}
  )
  _snake_case					     =       (model_prefix in flax_model_params) and (
      model_prefix not in {k.split('''.'''		)[0] for k in pt_state_dict.keys()}
  )

  # Need to change some parameters name to match Flax names
  for pt_key, pt_tensor in pt_state_dict.items():
    _snake_case					     =       tuple(pt_key.split('''.'''		)		)

    # remove base model prefix if necessary
    _snake_case					     =       pt_tuple_key[0] == model_prefix
    if load_model_with_head_into_base_model and has_base_model_prefix:
      _snake_case					     =       pt_tuple_key[1:]

    # Correctly rename weight parameters
    _snake_case					     =       rename_key_and_reshape_tensor(
        _lowerCAmelCase ,					_lowerCAmelCase ,					_lowerCAmelCase ,					_lowerCAmelCase		)

    # add model prefix if necessary
    _snake_case					     =       (model_prefix,) + flax_key in random_flax_state_dict
    if load_base_model_into_model_with_head and require_base_model_prefix:
      _snake_case					     =       (model_prefix,) + flax_key

    if flax_key in random_flax_state_dict:
      if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
        raise ValueError(
            f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
            f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)

        # add batch stats if the model contains batchnorm layers
    if "batch_stats" in flax_model.params:
      if "mean" in flax_key[-1] or "var" in flax_key[-1]:
        _snake_case					     =       jnp.asarray(_lowerCAmelCase		)
        continue
      # remove num_batches_tracked key
      if "num_batches_tracked" in flax_key[-1]:
        flax_state_dict.pop(_lowerCAmelCase ,					_lowerCAmelCase		)
        continue

      # also add unexpected weight so that warning is thrown
      _snake_case					     =       jnp.asarray(_lowerCAmelCase		)

    else:
      # also add unexpected weight so that warning is thrown
      _snake_case					     =       jnp.asarray(_lowerCAmelCase		)

  return unflatten_dict(_lowerCAmelCase		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Optional[int] ,					__lowerCamelCase   :       Any		)     ->					str:
  import torch

  # Load the index
  _snake_case					     =       {}
  for shard_file in shard_filenames:
    # load using msgpack utils
    _snake_case					     =       torch.load(_lowerCAmelCase		)
    _snake_case					     =       {k: v.numpy() for k, v in pt_state_dict.items()}

    _snake_case					     =       flax_model.base_model_prefix

    # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
    if "batch_stats" in flax_model.params:
      _snake_case					     =       flax_model.params['''params''']

      _snake_case					     =       flatten_dict(_lowerCAmelCase		)
      random_flax_state_dict.update(flatten_dict(flax_model.params['''batch_stats''']		)		)
    else:
      _snake_case					     =       flax_model.params
      _snake_case					     =       flatten_dict(_lowerCAmelCase		)

    _snake_case					     =       (model_prefix not in flax_model_params) and (
        model_prefix in {k.split('''.'''		)[0] for k in pt_state_dict.keys()}
    )
    _snake_case					     =       (model_prefix in flax_model_params) and (
        model_prefix not in {k.split('''.'''		)[0] for k in pt_state_dict.keys()}
    )
    # Need to change some parameters name to match Flax names
    for pt_key, pt_tensor in pt_state_dict.items():
      _snake_case					     =       tuple(pt_key.split('''.'''		)		)

      # remove base model prefix if necessary
      _snake_case					     =       pt_tuple_key[0] == model_prefix
      if load_model_with_head_into_base_model and has_base_model_prefix:
        _snake_case					     =       pt_tuple_key[1:]

      # Correctly rename weight parameters
      _snake_case					     =       rename_key_and_reshape_tensor(
          _lowerCAmelCase ,					_lowerCAmelCase ,					_lowerCAmelCase ,					_lowerCAmelCase		)
      # add model prefix if necessary
      _snake_case					     =       (model_prefix,) + flax_key in random_flax_state_dict
      if load_base_model_into_model_with_head and require_base_model_prefix:
        _snake_case					     =       (model_prefix,) + flax_key

      if flax_key in random_flax_state_dict:
        if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
          raise ValueError(
              f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
              f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)

            # add batch stats if the model contains batchnorm layers
      if "batch_stats" in flax_model.params:
        if "mean" in flax_key[-1]:
          _snake_case					     =       jnp.asarray(_lowerCAmelCase		)
          continue
        if "var" in flax_key[-1]:
          _snake_case					     =       jnp.asarray(_lowerCAmelCase		)
          continue
        # remove num_batches_tracked key
        if "num_batches_tracked" in flax_key[-1]:
          flax_state_dict.pop(_lowerCAmelCase ,					_lowerCAmelCase		)
          continue

        # also add unexpected weight so that warning is thrown
        _snake_case					     =       jnp.asarray(_lowerCAmelCase		)

      else:
        # also add unexpected weight so that warning is thrown
        _snake_case					     =       jnp.asarray(_lowerCAmelCase		)
  return unflatten_dict(_lowerCAmelCase		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Dict ,					__lowerCamelCase   :       List[Any]		)     ->					Tuple:
  _snake_case					     =       os.path.abspath(_lowerCAmelCase		)
  logger.info(f'''Loading Flax weights from {flax_checkpoint_path}'''		)

  # import correct flax class
  _snake_case					     =       getattr(_lowerCAmelCase ,					'''Flax''' + model.__class__.__name__		)

  # load flax weight dict
  with open(_lowerCAmelCase ,					'''rb'''		) as state_f:
    try:
      _snake_case					     =       from_bytes(_lowerCAmelCase ,					state_f.read()		)
    except UnpicklingError:
      raise EnvironmentError(f'''Unable to convert {flax_checkpoint_path} to Flax deserializable object. '''		)

  return load_flax_weights_in_pytorch_model(_lowerCAmelCase ,					_lowerCAmelCase		)

def 					_UpperCAmelCase      ( __lowerCamelCase   :       Optional[int] ,					__lowerCamelCase   :       str		)     ->					Any:

  try:
    import torch  # noqa: F401
  except ImportError:
    logger.error(
        '''Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see'''
        ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
        ''' instructions.'''		)
    raise

  # check if we have bf16 weights
  _snake_case					     =       flatten_dict(jax.tree_util.tree_map(lambda __lowerCamelCase		: x.dtype == jnp.bfloataa ,					_lowerCAmelCase		)		).values()
  if any(_lowerCAmelCase		):
    # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
    # and bf16 is not fully supported in PT yet.
    logger.warning(
        '''Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '''
        '''before loading those in PyTorch model.'''		)
    _snake_case					     =       jax.tree_util.tree_map(
        lambda __lowerCamelCase		: params.astype(np.floataa		) if params.dtype == jnp.bfloataa else params ,					_lowerCAmelCase		)

  _snake_case					     =       flatten_dict(_lowerCAmelCase		)
  _snake_case					     =       pt_model.state_dict()

  _snake_case					     =       (pt_model.base_model_prefix in flax_state) and (
      pt_model.base_model_prefix not in {k.split('''.'''		)[0] for k in pt_model_dict.keys()}
  )
  _snake_case					     =       (pt_model.base_model_prefix not in flax_state) and (
      pt_model.base_model_prefix in {k.split('''.'''		)[0] for k in pt_model_dict.keys()}
  )

  # keep track of unexpected & missing keys
  _snake_case					     =       []
  _snake_case					     =       set(pt_model_dict.keys()		)

  for flax_key_tuple, flax_tensor in flax_state_dict.items():
    _snake_case					     =       flax_key_tuple[0] == pt_model.base_model_prefix
    _snake_case					     =       '''.'''.join((pt_model.base_model_prefix,) + flax_key_tuple		) in pt_model_dict

    # adapt flax_key to prepare for loading from/to base model only
    if load_model_with_head_into_base_model and has_base_model_prefix:
      _snake_case					     =       flax_key_tuple[1:]
    elif load_base_model_into_model_with_head and require_base_model_prefix:
      _snake_case					     =       (pt_model.base_model_prefix,) + flax_key_tuple

    # rename flax weights to PyTorch format
    if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(_lowerCAmelCase		) not in pt_model_dict:
      # conv layer
      _snake_case					     =       flax_key_tuple[:-1] + ('''weight''',)
      _snake_case					     =       jnp.transpose(_lowerCAmelCase ,					(3, 2, 0, 1)		)
    elif flax_key_tuple[-1] == "kernel" and ".".join(_lowerCAmelCase		) not in pt_model_dict:
      # linear layer
      _snake_case					     =       flax_key_tuple[:-1] + ('''weight''',)
      _snake_case					     =       flax_tensor.T
    elif flax_key_tuple[-1] in ["scale", "embedding"]:
      _snake_case					     =       flax_key_tuple[:-1] + ('''weight''',)

    # adding batch stats from flax batch norm to pt
    elif "mean" in flax_key_tuple[-1]:
      _snake_case					     =       flax_key_tuple[:-1] + ('''running_mean''',)
    elif "var" in flax_key_tuple[-1]:
      _snake_case					     =       flax_key_tuple[:-1] + ('''running_var''',)

    if "batch_stats" in flax_state:
      _snake_case					     =       '''.'''.join(flax_key_tuple[1:]		)  # Remove the params/batch_stats header
    else:
      _snake_case					     =       '''.'''.join(_lowerCAmelCase		)

    # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
    _snake_case					     =       {}
    # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
    for key in pt_model_dict:
      _snake_case					     =       key.split('''.'''		)
      _snake_case					     =       None
      if key_components[-3::2] == ["parametrizations", "original0"]:
        _snake_case					     =       key_components[-2] + '''_g'''
      elif key_components[-3::2] == ["parametrizations", "original1"]:
        _snake_case					     =       key_components[-2] + '''_v'''
      if name is not None:
        _snake_case					     =       key_components[:-3] + [name]
        _snake_case					     =       '''.'''.join(_lowerCAmelCase		)
        _snake_case					     =       key

    if flax_key in special_pt_names:
      _snake_case					     =       special_pt_names[flax_key]

    if flax_key in pt_model_dict:
      if flax_tensor.shape != pt_model_dict[flax_key].shape:
        raise ValueError(
            f'''Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '''
            f'''to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)
      else:
        # add weight to pytorch dict
        _snake_case					     =       np.asarray(_lowerCAmelCase		) if not isinstance(_lowerCAmelCase ,					np.ndarray		) else flax_tensor
        _snake_case					     =       torch.from_numpy(_lowerCAmelCase		)
        # remove from missing keys
        missing_keys.remove(_lowerCAmelCase		)
    else:
      # weight is not expected by PyTorch model
      unexpected_keys.append(_lowerCAmelCase		)

  pt_model.load_state_dict(_lowerCAmelCase		)

  # re-transform missing_keys to list
  _snake_case					     =       list(_lowerCAmelCase		)

  if len(_lowerCAmelCase		) > 0:
    logger.warning(
        '''Some weights of the Flax model were not used when initializing the PyTorch model'''
        f''' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'''
        f''' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'''
        ''' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'''
        f''' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'''
        ''' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'''
        ''' FlaxBertForSequenceClassification model).'''		)
  else:
    logger.warning(f'''All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n'''		)
  if len(_lowerCAmelCase		) > 0:
    logger.warning(
        f'''Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'''
        f''' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'''
        ''' use it for predictions and inference.'''		)
  else:
    logger.warning(
        f'''All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n'''
        '''If your task is similar to the task the model of the checkpoint was trained on, '''
        f'''you can already use {pt_model.__class__.__name__} for predictions without further training.'''		)

  return pt_model








'''simple docstring'''






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Optional[Any]  )      ->   Optional[int]:
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 4
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_1", "col_2", "col_3"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,      [False, True]  )
def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Optional[int]  )      ->   Dict:
							UpperCAmelCase :       Dict 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[Any] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCAmelCase :       str 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase   ,      keep_in_memory=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						int  )      ->   Optional[int]:
							UpperCAmelCase :       Any 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Tuple 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Any 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       List[Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Dict 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						List[Any]   ,      _lowerCAmelCase  :						Tuple  )      ->   Tuple:
							UpperCAmelCase :       Optional[Any] 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Optional[int] 							=		{'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
							UpperCAmelCase :       int 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       Any 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_3", "col_1", "col_2"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Dict  )      ->   Union[str, Any]:
							# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
							UpperCAmelCase :       Tuple 							=		{'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
							UpperCAmelCase :       List[str] 							=		features.copy()
							UpperCAmelCase :       Union[str, Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[str] 							=		JsonDatasetReader(_lowerCAmelCase   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_2", "col_3", "col_1"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''split'''   ,      [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def  snake_case_     (					_lowerCAmelCase  :						Union[str, Any]   ,      _lowerCAmelCase  :						Tuple   ,      _lowerCAmelCase  :						List[str]  )      ->   Optional[Any]:
							UpperCAmelCase :       Any 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[Any] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       List[Any] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase   ,      split=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )
							assert dataset.split == split if split else "train"


@pytest.mark.parametrize('''path_type'''   ,      [str, list]  )
def  snake_case_     (					_lowerCAmelCase  :						List[Any]   ,      _lowerCAmelCase  :						str   ,      _lowerCAmelCase  :						Any  )      ->   Dict:
							if issubclass(_lowerCAmelCase   ,      _lowerCAmelCase  ):
														UpperCAmelCase :       str 							=		jsonl_path
							elif issubclass(_lowerCAmelCase   ,      _lowerCAmelCase  ):
														UpperCAmelCase :       Dict 							=		[jsonl_path]
							UpperCAmelCase :       int 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Dict 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[int] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_dataset(_lowerCAmelCase   ,      _lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Dict   ,      _lowerCAmelCase  :						Dict=("train",)  )      ->   Union[str, Any]:
							assert isinstance(_lowerCAmelCase   ,      _lowerCAmelCase  )
							for split in splits:
														UpperCAmelCase :       List[str] 							=		dataset_dict[split]
														assert dataset.num_rows == 4
														assert dataset.num_columns == 3
														assert dataset.column_names == ["col_1", "col_2", "col_3"]
														for feature, expected_dtype in expected_features.items():
																					assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize('''keep_in_memory'''   ,      [False, True]  )
def  snake_case_     (					_lowerCAmelCase  :						int   ,      _lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						str  )      ->   Any:
							UpperCAmelCase :       Optional[Any] 							=		tmp_path / '''cache'''
							UpperCAmelCase :       List[str] 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCAmelCase :       Optional[int] 							=		JsonDatasetReader({'''train''': jsonl_path}   ,      cache_dir=_lowerCAmelCase   ,      keep_in_memory=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize(
    '''features'''   ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]   ,      )
def  snake_case_     (					_lowerCAmelCase  :						Optional[Any]   ,      _lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Union[str, Any]  )      ->   int:
							UpperCAmelCase :       Dict 							=		tmp_path / '''cache'''
							UpperCAmelCase :       Dict 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[int] 							=		features.copy() if features else default_expected_features
							UpperCAmelCase :       Union[str, Any] 							=		(
							    Features({feature: Value(_lowerCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
							)
							UpperCAmelCase :       Tuple 							=		JsonDatasetReader({'''train''': jsonl_path}   ,      features=_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase  )


@pytest.mark.parametrize('''split'''   ,      [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def  snake_case_     (					_lowerCAmelCase  :						Any   ,      _lowerCAmelCase  :						Optional[int]   ,      _lowerCAmelCase  :						Dict  )      ->   Union[str, Any]:
							if split:
														UpperCAmelCase :       Optional[int] 							=		{split: jsonl_path}
							else:
														UpperCAmelCase :       Any 							=		'''train'''
														UpperCAmelCase :       Any 							=		{'''train''': jsonl_path, '''test''': jsonl_path}
							UpperCAmelCase :       Tuple 							=		tmp_path / '''cache'''
							UpperCAmelCase :       int 							=		{'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
							UpperCAmelCase :       Optional[Any] 							=		JsonDatasetReader(_lowerCAmelCase   ,      cache_dir=_lowerCAmelCase  ).read()
							_check_json_datasetdict(_lowerCAmelCase   ,      _lowerCAmelCase   ,      splits=list(path.keys()  )  )
							assert all(dataset[split].split == split for split in path.keys()  )


def  snake_case_     (					_lowerCAmelCase  :						List[str]  )      ->   str:
							return json.load(_lowerCAmelCase  )


def  snake_case_     (					_lowerCAmelCase  :						Dict  )      ->   str:
							return [json.loads(_lowerCAmelCase  ) for line in buffer]




class 				SCREAMING_SNAKE_CASE:







	"""simple docstring"""






	@pytest.mark.parametrize('''lines, load_json_function'''		,			[(True, load_json_lines), (False, load_json)]		)
	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     Tuple		,			__snake_case			:     Optional[Any]		,			__snake_case			:     Optional[int]		)			->		Dict:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Union[str, Any] 							=		load_json_function(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								assert isinstance(exported_content[0]		,			__snake_case		)
								assert len(__snake_case		) == 10







	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''		,			[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''		), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]		,			)
	def      A	(						self			:     Optional[int]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     str		,			__snake_case			:     Optional[Any]		,			__snake_case			:     List[str]		,			__snake_case			:     Optional[Any]		)			->		List[Any]:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			orient=__snake_case		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Union[str, Any] 							=		load_json(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								if keys:
															if container is dict:
																						assert exported_content.keys() == keys
															else:
																						assert exported_content[0].keys() == keys
								else:
															assert not hasattr(__snake_case		,			'''keys'''		) and not hasattr(exported_content[0]		,			'''keys'''		)
								if len_at:
															assert len(exported_content[len_at]		) == 10
								else:
															assert len(__snake_case		) == 10







	@pytest.mark.parametrize('''lines, load_json_function'''		,			[(True, load_json_lines), (False, load_json)]		)
	def      A	(						self			:     str		,			__snake_case			:     str		,			__snake_case			:     str		,			__snake_case			:     int		)			->		Any:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			num_proc=2		).write()
															buffer.seek(0		)
															UpperCAmelCase :       Any 							=		load_json_function(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								assert isinstance(exported_content[0]		,			__snake_case		)
								assert len(__snake_case		) == 10







	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''		,			[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''		), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]		,			)
	def      A	(						self			:     Any		,			__snake_case			:     int		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Optional[int]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     List[str]		)			->		Any:
								with io.BytesIO() as buffer:
															JsonDatasetWriter(__snake_case		,			__snake_case		,			lines=__snake_case		,			orient=__snake_case		,			num_proc=2		).write()
															buffer.seek(0		)
															UpperCAmelCase :       List[str] 							=		load_json(__snake_case		)
								assert isinstance(__snake_case		,			__snake_case		)
								if keys:
															if container is dict:
																						assert exported_content.keys() == keys
															else:
																						assert exported_content[0].keys() == keys
								else:
															assert not hasattr(__snake_case		,			'''keys'''		) and not hasattr(exported_content[0]		,			'''keys'''		)
								if len_at:
															assert len(exported_content[len_at]		) == 10
								else:
															assert len(__snake_case		) == 10







	def      A	(						self			:     List[Any]		,			__snake_case			:     str		)			->		Dict:
								with pytest.raises(__snake_case		):
															with io.BytesIO() as buffer:
																						JsonDatasetWriter(__snake_case		,			__snake_case		,			num_proc=0		)







	@pytest.mark.parametrize('''compression, extension'''		,			[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]		)
	def      A	(						self			:     Optional[int]		,			__snake_case			:     Any		,			__snake_case			:     str		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Union[str, Any]		,			__snake_case			:     Dict		)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		tmp_path_factory.mktemp('''data'''		) / F"""test.json.{extension}"""
								UpperCAmelCase :       List[Any] 							=		str(shared_datadir / F"""test_file.json.{extension}"""		)
								JsonDatasetWriter(__snake_case		,			__snake_case		,			compression=__snake_case		).write()

								with fsspec.open(__snake_case		,			'''rb'''		,			compression='''infer'''		) as f:
															UpperCAmelCase :       str 							=		f.read()
								with fsspec.open(__snake_case		,			'''rb'''		,			compression='''infer'''		) as f:
															UpperCAmelCase :       Optional[int] 							=		f.read()
								assert exported_content == original_content





