


import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Tuple			,     SCREAMING_SNAKE_CASE__			:					int				):
     assert isinstance(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)
     assert dataset.num_rows == 4
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_1", "col_2", "col_3"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'			,     [False, True]				)
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					str			,     SCREAMING_SNAKE_CASE__			:					int			,     SCREAMING_SNAKE_CASE__			:					Tuple				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__			,     keep_in_memory=SCREAMING_SNAKE_CASE__				).read()
     _check_json_dataset(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

@pytest.mark.parametrize(
    'features'			,     [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]			,     )
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Dict			,     SCREAMING_SNAKE_CASE__			:					Any			,     SCREAMING_SNAKE_CASE__			:					List[str]				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     __UpperCamelCase    							=features.copy() if features else default_expected_features
     __UpperCamelCase    							=(
         Features({feature: Value(SCREAMING_SNAKE_CASE__				) for feature, dtype in features.items()}				) if features is not None else None
     )
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     features=SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     _check_json_dataset(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

@pytest.mark.parametrize(
    'features'			,     [
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]			,     )
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Tuple			,     SCREAMING_SNAKE_CASE__			:					Union[str, Any]			,     SCREAMING_SNAKE_CASE__			:					List[str]				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
     __UpperCamelCase    							=features.copy() if features else default_expected_features
     __UpperCamelCase    							=(
         Features({feature: Value(SCREAMING_SNAKE_CASE__				) for feature, dtype in features.items()}				) if features is not None else None
     )
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     features=SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     assert isinstance(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)
     assert dataset.num_rows == 2
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_3", "col_1", "col_2"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					str			,     SCREAMING_SNAKE_CASE__			:					int				):
     # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
     __UpperCamelCase    							={'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
     __UpperCamelCase    							=features.copy()
     __UpperCamelCase    							=(
         Features({feature: Value(SCREAMING_SNAKE_CASE__				) for feature, dtype in features.items()}				) if features is not None else None
     )
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     features=SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     assert isinstance(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)
     assert dataset.num_rows == 2
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_2", "col_3", "col_1"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('split'			,     [None, NamedSplit('train'				), 'train', 'test']				)
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Any			,     SCREAMING_SNAKE_CASE__			:					Optional[int]			,     SCREAMING_SNAKE_CASE__			:					Any				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__			,     split=SCREAMING_SNAKE_CASE__				).read()
     _check_json_dataset(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)
     assert dataset.split == split if split else "train"

@pytest.mark.parametrize('path_type'			,     [str, list]				)
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Any			,     SCREAMING_SNAKE_CASE__			:					List[Any]			,     SCREAMING_SNAKE_CASE__			:					int				):
     if issubclass(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				):
          __UpperCamelCase    							=jsonl_path
     elif issubclass(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				):
          __UpperCamelCase    							=[jsonl_path]
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     _check_json_dataset(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					int			,     SCREAMING_SNAKE_CASE__			:					List[str]			,     SCREAMING_SNAKE_CASE__			:					Optional[Any]=("train",)				):
     assert isinstance(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)
     for split in splits:
          __UpperCamelCase    							=dataset_dict[split]
          assert dataset.num_rows == 4
          assert dataset.num_columns == 3
          assert dataset.column_names == ["col_1", "col_2", "col_3"]
          for feature, expected_dtype in expected_features.items():
               assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'			,     [False, True]				)
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Union[str, Any]			,     SCREAMING_SNAKE_CASE__			:					Tuple			,     SCREAMING_SNAKE_CASE__			:					List[Any]				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          __UpperCamelCase    							=JsonDatasetReader({'train': jsonl_path}			,     cache_dir=SCREAMING_SNAKE_CASE__			,     keep_in_memory=SCREAMING_SNAKE_CASE__				).read()
     _check_json_datasetdict(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

@pytest.mark.parametrize(
    'features'			,     [
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]			,     )
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					List[Any]			,     SCREAMING_SNAKE_CASE__			:					Tuple			,     SCREAMING_SNAKE_CASE__			:					Any				):
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     __UpperCamelCase    							=features.copy() if features else default_expected_features
     __UpperCamelCase    							=(
         Features({feature: Value(SCREAMING_SNAKE_CASE__				) for feature, dtype in features.items()}				) if features is not None else None
     )
     __UpperCamelCase    							=JsonDatasetReader({'train': jsonl_path}			,     features=SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     _check_json_datasetdict(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__				)

@pytest.mark.parametrize('split'			,     [None, NamedSplit('train'				), 'train', 'test']				)
def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Any			,     SCREAMING_SNAKE_CASE__			:					int			,     SCREAMING_SNAKE_CASE__			:					Union[str, Any]				):
     if split:
          __UpperCamelCase    							={split: jsonl_path}
     else:
          __UpperCamelCase    							='train'
          __UpperCamelCase    							={'train': jsonl_path, 'test': jsonl_path}
     __UpperCamelCase    							=tmp_path / 'cache'
     __UpperCamelCase    							={'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
     __UpperCamelCase    							=JsonDatasetReader(SCREAMING_SNAKE_CASE__			,     cache_dir=SCREAMING_SNAKE_CASE__				).read()
     _check_json_datasetdict(SCREAMING_SNAKE_CASE__			,     SCREAMING_SNAKE_CASE__			,     splits=list(path.keys()				)				)
     assert all(dataset[split].split == split for split in path.keys()				)

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Union[str, Any]				):
     return json.load(SCREAMING_SNAKE_CASE__				)

def 						_UpperCAmelCase				(			SCREAMING_SNAKE_CASE__			:					Any				):
     return [json.loads(SCREAMING_SNAKE_CASE__				) for line in buffer]






class     UpperCAmelCase__       :



    """simple docstring"""






    @pytest.mark.parametrize('lines, load_json_function'					,   [(True, load_json_lines), (False, load_json)]    )
    def _a					(			self					,   A_					,   A_					,   A_    )  ->       List[str]:
         with io.BytesIO() as buffer:
              JsonDatasetWriter(A_					,   A_					,   lines=A_    ).write()
              buffer.seek(0    )
              __UpperCamelCase    							=load_json_function(A_    )
         assert isinstance(A_					,   A_    )
         assert isinstance(exported_content[0]					,   A_    )
         assert len(A_    ) == 10






    @pytest.mark.parametrize(
        'orient, container, keys, len_at'					,   [
            ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
            ('split', dict, {'columns', 'data'}, 'data'),
            ('index', dict, set('0123456789'    ), None),
            ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
            ('values', list, None, None),
            ('table', dict, {'schema', 'data'}, 'data'),
        ]					,   )
    def _a					(			self					,   A_					,   A_					,   A_					,   A_					,   A_    )  ->       Optional[int]:
         with io.BytesIO() as buffer:
              JsonDatasetWriter(A_					,   A_					,   lines=A_					,   orient=A_    ).write()
              buffer.seek(0    )
              __UpperCamelCase    							=load_json(A_    )
         assert isinstance(A_					,   A_    )
         if keys:
              if container is dict:
                   assert exported_content.keys() == keys
              else:
                   assert exported_content[0].keys() == keys
         else:
              assert not hasattr(A_					,   'keys'    ) and not hasattr(exported_content[0]					,   'keys'    )
         if len_at:
              assert len(exported_content[len_at]    ) == 10
         else:
              assert len(A_    ) == 10






    @pytest.mark.parametrize('lines, load_json_function'					,   [(True, load_json_lines), (False, load_json)]    )
    def _a					(			self					,   A_					,   A_					,   A_    )  ->       List[Any]:
         with io.BytesIO() as buffer:
              JsonDatasetWriter(A_					,   A_					,   lines=A_					,   num_proc=2    ).write()
              buffer.seek(0    )
              __UpperCamelCase    							=load_json_function(A_    )
         assert isinstance(A_					,   A_    )
         assert isinstance(exported_content[0]					,   A_    )
         assert len(A_    ) == 10






    @pytest.mark.parametrize(
        'orient, container, keys, len_at'					,   [
            ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
            ('split', dict, {'columns', 'data'}, 'data'),
            ('index', dict, set('0123456789'    ), None),
            ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
            ('values', list, None, None),
            ('table', dict, {'schema', 'data'}, 'data'),
        ]					,   )
    def _a					(			self					,   A_					,   A_					,   A_					,   A_					,   A_    )  ->       Tuple:
         with io.BytesIO() as buffer:
              JsonDatasetWriter(A_					,   A_					,   lines=A_					,   orient=A_					,   num_proc=2    ).write()
              buffer.seek(0    )
              __UpperCamelCase    							=load_json(A_    )
         assert isinstance(A_					,   A_    )
         if keys:
              if container is dict:
                   assert exported_content.keys() == keys
              else:
                   assert exported_content[0].keys() == keys
         else:
              assert not hasattr(A_					,   'keys'    ) and not hasattr(exported_content[0]					,   'keys'    )
         if len_at:
              assert len(exported_content[len_at]    ) == 10
         else:
              assert len(A_    ) == 10






    def _a					(			self					,   A_    )  ->       Optional[Any]:
         with pytest.raises(A_    ):
              with io.BytesIO() as buffer:
                   JsonDatasetWriter(A_					,   A_					,   num_proc=0    )






    @pytest.mark.parametrize('compression, extension'					,   [('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]    )
    def _a					(			self					,   A_					,   A_					,   A_					,   A_					,   A_    )  ->       int:
         __UpperCamelCase    							=tmp_path_factory.mktemp('data'    ) / f'test.json.{extension}'
         __UpperCamelCase    							=str(shared_datadir / f'test_file.json.{extension}'    )
         JsonDatasetWriter(A_					,   A_					,   compression=A_    ).write()

         with fsspec.open(A_					,   'rb'					,   compression='infer'    ) as f:
              __UpperCamelCase    							=f.read()
         with fsspec.open(A_					,   'rb'					,   compression='infer'    ) as f:
              __UpperCamelCase    							=f.read()
         assert exported_content == original_content









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A		       =						{
    'configuration_convbert': ['CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ConvBertConfig', 'ConvBertOnnxConfig'],
    'tokenization_convbert': ['ConvBertTokenizer'],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A		       =						['ConvBertTokenizerFast']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A		       =						[
       'CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'ConvBertForMaskedLM',
       'ConvBertForMultipleChoice',
       'ConvBertForQuestionAnswering',
       'ConvBertForSequenceClassification',
       'ConvBertForTokenClassification',
       'ConvBertLayer',
       'ConvBertModel',
       'ConvBertPreTrainedModel',
       'load_tf_weights_in_convbert',
   ]


try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A		       =						[
       'TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TFConvBertForMaskedLM',
       'TFConvBertForMultipleChoice',
       'TFConvBertForQuestionAnswering',
       'TFConvBertForSequenceClassification',
       'TFConvBertForTokenClassification',
       'TFConvBertLayer',
       'TFConvBertModel',
       'TFConvBertPreTrainedModel',
   ]


if TYPE_CHECKING:
   from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
   from .tokenization_convbert import ConvBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_convbert_fast import ConvBertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_convbert import (
          CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          ConvBertForMaskedLM,
          ConvBertForMultipleChoice,
          ConvBertForQuestionAnswering,
          ConvBertForSequenceClassification,
          ConvBertForTokenClassification,
          ConvBertLayer,
          ConvBertModel,
          ConvBertPreTrainedModel,
          load_tf_weights_in_convbert,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_convbert import (
          TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFConvBertForMaskedLM,
          TFConvBertForMultipleChoice,
          TFConvBertForQuestionAnswering,
          TFConvBertForSequenceClassification,
          TFConvBertForTokenClassification,
          TFConvBertLayer,
          TFConvBertModel,
          TFConvBertPreTrainedModel,
      )


else:
   import sys

   _A		       =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





