





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def      _A       (		SCREAMING_SNAKE_CASE     :   Tuple       ,       SCREAMING_SNAKE_CASE     :   int						):





  """simple docstring"""



  assert isinstance(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"       ,       [False, True]						)
def      _A       (		SCREAMING_SNAKE_CASE     :   str       ,       SCREAMING_SNAKE_CASE     :   int       ,       SCREAMING_SNAKE_CASE     :   Tuple						):





  """simple docstring"""



  a__	:					Dict										=tmp_path / "cache"
  a__	:					Tuple										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    a__	:					Union[str, Any]										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__       ,       keep_in_memory=SCREAMING_SNAKE_CASE__						).read()
  _check_json_dataset(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
@pytest.mark.parametrize(
    "features"       ,       [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]       ,       )
def      _A       (		SCREAMING_SNAKE_CASE     :   Dict       ,       SCREAMING_SNAKE_CASE     :   Any       ,       SCREAMING_SNAKE_CASE     :   List[str]						):





  """simple docstring"""



  a__	:					Optional[Any]										=tmp_path / "cache"
  a__	:					Optional[Any]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  a__	:					Optional[Any]										=features.copy() if features else default_expected_features
  a__	:					Any										=(
      Features({feature: Value(SCREAMING_SNAKE_CASE__						) for feature, dtype in features.items()}						) if features is not None else None
  )
  a__	:					Optional[int]										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       features=SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  _check_json_dataset(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
@pytest.mark.parametrize(
    "features"       ,       [
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]       ,       )
def      _A       (		SCREAMING_SNAKE_CASE     :   Tuple       ,       SCREAMING_SNAKE_CASE     :   Union[str, Any]       ,       SCREAMING_SNAKE_CASE     :   List[str]						):





  """simple docstring"""



  a__	:					int										=tmp_path / "cache"
  a__	:					Tuple										={"col_3": "float64", "col_1": "string", "col_2": "int64"}
  a__	:					Any										=features.copy() if features else default_expected_features
  a__	:					int										=(
      Features({feature: Value(SCREAMING_SNAKE_CASE__						) for feature, dtype in features.items()}						) if features is not None else None
  )
  a__	:					Dict										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       features=SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  assert isinstance(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_3", "col_1", "col_2"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
def      _A       (		SCREAMING_SNAKE_CASE     :   str       ,       SCREAMING_SNAKE_CASE     :   int						):





  """simple docstring"""



  a__	:					str										={"col_2": "int64", "col_3": "float64", "col_1": "string"}
  a__	:					Union[str, Any]										=features.copy()
  a__	:					Any										=(
      Features({feature: Value(SCREAMING_SNAKE_CASE__						) for feature, dtype in features.items()}						) if features is not None else None
  )
  a__	:					str										=tmp_path / "cache"
  a__	:					int										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       features=SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  assert isinstance(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_2", "col_3", "col_1"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("split"       ,       [None, NamedSplit("train"						), "train", "test"]						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Any       ,       SCREAMING_SNAKE_CASE     :   Optional[int]       ,       SCREAMING_SNAKE_CASE     :   Any						):





  """simple docstring"""



  a__	:					List[Any]										=tmp_path / "cache"
  a__	:					Optional[int]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  a__	:					Optional[Any]										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__       ,       split=SCREAMING_SNAKE_CASE__						).read()
  _check_json_dataset(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
  assert dataset.split == split if split else "train"
@pytest.mark.parametrize("path_type"       ,       [str, list]						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Any       ,       SCREAMING_SNAKE_CASE     :   List[Any]       ,       SCREAMING_SNAKE_CASE     :   int						):





  """simple docstring"""



  if issubclass(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						):
    a__	:					Optional[int]										=jsonl_path
  elif issubclass(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						):
    a__	:					Tuple										=[jsonl_path]
  a__	:					Union[str, Any]										=tmp_path / "cache"
  a__	:					Optional[int]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  a__	:					Optional[int]										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  _check_json_dataset(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
def      _A       (		SCREAMING_SNAKE_CASE     :   int       ,       SCREAMING_SNAKE_CASE     :   List[str]       ,       SCREAMING_SNAKE_CASE     :   Optional[Any]=("train",)						):





  """simple docstring"""



  assert isinstance(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
  for split in splits:
    a__	:					Union[str, Any]										=dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"       ,       [False, True]						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Union[str, Any]       ,       SCREAMING_SNAKE_CASE     :   Tuple       ,       SCREAMING_SNAKE_CASE     :   List[Any]						):





  """simple docstring"""



  a__	:					str										=tmp_path / "cache"
  a__	:					Optional[int]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    a__	:					str										=JsonDatasetReader({"train": jsonl_path}       ,       cache_dir=SCREAMING_SNAKE_CASE__       ,       keep_in_memory=SCREAMING_SNAKE_CASE__						).read()
  _check_json_datasetdict(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
@pytest.mark.parametrize(
    "features"       ,       [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]       ,       )
def      _A       (		SCREAMING_SNAKE_CASE     :   List[Any]       ,       SCREAMING_SNAKE_CASE     :   Tuple       ,       SCREAMING_SNAKE_CASE     :   Any						):





  """simple docstring"""



  a__	:					Tuple										=tmp_path / "cache"
  a__	:					Union[str, Any]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  a__	:					str										=features.copy() if features else default_expected_features
  a__	:					Tuple										=(
      Features({feature: Value(SCREAMING_SNAKE_CASE__						) for feature, dtype in features.items()}						) if features is not None else None
  )
  a__	:					Dict										=JsonDatasetReader({"train": jsonl_path}       ,       features=SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  _check_json_datasetdict(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__						)
@pytest.mark.parametrize("split"       ,       [None, NamedSplit("train"						), "train", "test"]						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Any       ,       SCREAMING_SNAKE_CASE     :   int       ,       SCREAMING_SNAKE_CASE     :   Union[str, Any]						):





  """simple docstring"""



  if split:
    a__	:					List[Any]										={split: jsonl_path}
  else:
    a__	:					Union[str, Any]										="train"
    a__	:					int										={"train": jsonl_path, "test": jsonl_path}
  a__	:					Dict										=tmp_path / "cache"
  a__	:					List[Any]										={"col_1": "string", "col_2": "int64", "col_3": "float64"}
  a__	:					Tuple										=JsonDatasetReader(SCREAMING_SNAKE_CASE__       ,       cache_dir=SCREAMING_SNAKE_CASE__						).read()
  _check_json_datasetdict(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__       ,       splits=list(path.keys()						)						)
  assert all(dataset[split].split == split for split in path.keys()						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Union[str, Any]						):





  """simple docstring"""



  return json.load(SCREAMING_SNAKE_CASE__						)
def      _A       (		SCREAMING_SNAKE_CASE     :   Any						):





  """simple docstring"""



  return [json.loads(SCREAMING_SNAKE_CASE__						) for line in buffer]







class 				__lowerCAmelCase       :







      @pytest.mark.parametrize("lines, load_json_function"		,      [(True, load_json_lines), (False, load_json)]							)
      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						List[str]:



        '''simple docstring'''




        with io.BytesIO() as buffer:
          JsonDatasetWriter(A_		,      A_		,      lines=A_							).write()
          buffer.seek(0							)
          a__	:					Any										=load_json_function(A_							)
        assert isinstance(A_		,      A_							)
        assert isinstance(exported_content[0]		,      A_							)
        assert len(A_							) == 1_0







      @pytest.mark.parametrize(
          "orient, container, keys, len_at"		,      [
              ("records", list, {"tokens", "labels", "answers", "id"}, None),
              ("split", dict, {"columns", "data"}, "data"),
              ("index", dict, set("0123456789"							), None),
              ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
              ("values", list, None, None),
              ("table", dict, {"schema", "data"}, "data"),
          ]		,      )
      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Optional[int]:



        '''simple docstring'''




        with io.BytesIO() as buffer:
          JsonDatasetWriter(A_		,      A_		,      lines=A_		,      orient=A_							).write()
          buffer.seek(0							)
          a__	:					Any										=load_json(A_							)
        assert isinstance(A_		,      A_							)
        if keys:
          if container is dict:
            assert exported_content.keys() == keys
          else:
            assert exported_content[0].keys() == keys
        else:
          assert not hasattr(A_		,      "keys"							) and not hasattr(exported_content[0]		,      "keys"							)
        if len_at:
          assert len(exported_content[len_at]							) == 1_0
        else:
          assert len(A_							) == 1_0







      @pytest.mark.parametrize("lines, load_json_function"		,      [(True, load_json_lines), (False, load_json)]							)
      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						List[Any]:



        '''simple docstring'''




        with io.BytesIO() as buffer:
          JsonDatasetWriter(A_		,      A_		,      lines=A_		,      num_proc=2							).write()
          buffer.seek(0							)
          a__	:					Tuple										=load_json_function(A_							)
        assert isinstance(A_		,      A_							)
        assert isinstance(exported_content[0]		,      A_							)
        assert len(A_							) == 1_0







      @pytest.mark.parametrize(
          "orient, container, keys, len_at"		,      [
              ("records", list, {"tokens", "labels", "answers", "id"}, None),
              ("split", dict, {"columns", "data"}, "data"),
              ("index", dict, set("0123456789"							), None),
              ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
              ("values", list, None, None),
              ("table", dict, {"schema", "data"}, "data"),
          ]		,      )
      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Tuple:



        '''simple docstring'''




        with io.BytesIO() as buffer:
          JsonDatasetWriter(A_		,      A_		,      lines=A_		,      orient=A_		,      num_proc=2							).write()
          buffer.seek(0							)
          a__	:					Optional[Any]										=load_json(A_							)
        assert isinstance(A_		,      A_							)
        if keys:
          if container is dict:
            assert exported_content.keys() == keys
          else:
            assert exported_content[0].keys() == keys
        else:
          assert not hasattr(A_		,      "keys"							) and not hasattr(exported_content[0]		,      "keys"							)
        if len_at:
          assert len(exported_content[len_at]							) == 1_0
        else:
          assert len(A_							) == 1_0







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        with pytest.raises(A_							):
          with io.BytesIO() as buffer:
            JsonDatasetWriter(A_		,      A_		,      num_proc=0							)







      @pytest.mark.parametrize("compression, extension"		,      [("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]							)
      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						int:



        '''simple docstring'''




        a__	:					Optional[Any]										=tmp_path_factory.mktemp("data"							) / F'''test.json.{extension}'''
        a__	:					Any										=str(shared_datadir / F'''test_file.json.{extension}'''							)
        JsonDatasetWriter(A_		,      A_		,      compression=A_							).write()

        with fsspec.open(A_		,      "rb"		,      compression="infer"							) as f:
          a__	:					Dict										=f.read()
        with fsspec.open(A_		,      "rb"		,      compression="infer"							) as f:
          a__	:					str										=f.read()
        assert exported_content == original_content










import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


_A		       =						logging.get_logger(__name__)

_A		       =						{
    'Salesforce/instruct-blip-flan-t5': 'https://huggingface.co/Salesforce/instruct-blip-flan-t5/resolve/main/config.json',
}






class     UpperCAmelCase__       (					A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Optional[int]      =  "instructblip_vision_model"






    def __init__(			self					,   A_=1408					,   A_=6144					,   A_=39					,   A_=16					,   A_=224					,   A_=14					,   A_="gelu"					,   A_=1E-6					,   A_=0.0					,   A_=1E-10					,   A_=True					,   **A_					,   )  ->       Tuple:
         super().__init__(**A_    )

         __UpperCamelCase    							=hidden_size
         __UpperCamelCase    							=intermediate_size
         __UpperCamelCase    							=num_hidden_layers
         __UpperCamelCase    							=num_attention_heads
         __UpperCamelCase    							=patch_size
         __UpperCamelCase    							=image_size
         __UpperCamelCase    							=initializer_range
         __UpperCamelCase    							=attention_dropout
         __UpperCamelCase    							=layer_norm_eps
         __UpperCamelCase    							=hidden_act
         __UpperCamelCase    							=qkv_bias






    @classmethod
    def _a					(			cls					,   A_					,   **A_    )  ->       "PretrainedConfig":
         cls._set_token_in_kwargs(A_    )

         __UpperCamelCase							,      __UpperCamelCase    							=cls.get_config_dict(A_					,   **A_    )

         # get the vision config dict if we are loading from InstructBlipConfig
         if config_dict.get('model_type'    ) == "instructblip":
              __UpperCamelCase    							=config_dict['vision_config']

         if "model_type" in config_dict and hasattr(cls					,   'model_type'    ) and config_dict["model_type"] != cls.model_type:
              logger.warning(
                  f'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                  f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'    )

         return cls.from_dict(A_					,   **A_    )






class     UpperCAmelCase__       (					A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Optional[Any]      =  "instructblip_qformer"






    def __init__(			self					,   A_=30522					,   A_=768					,   A_=12					,   A_=12					,   A_=3072					,   A_="gelu"					,   A_=0.1					,   A_=0.1					,   A_=512					,   A_=0.02					,   A_=1E-12					,   A_=0					,   A_="absolute"					,   A_=2					,   A_=1408					,   **A_					,   )  ->       Optional[Any]:
         super().__init__(pad_token_id=A_					,   **A_    )

         __UpperCamelCase    							=vocab_size
         __UpperCamelCase    							=hidden_size
         __UpperCamelCase    							=num_hidden_layers
         __UpperCamelCase    							=num_attention_heads
         __UpperCamelCase    							=hidden_act
         __UpperCamelCase    							=intermediate_size
         __UpperCamelCase    							=hidden_dropout_prob
         __UpperCamelCase    							=attention_probs_dropout_prob
         __UpperCamelCase    							=max_position_embeddings
         __UpperCamelCase    							=initializer_range
         __UpperCamelCase    							=layer_norm_eps
         __UpperCamelCase    							=position_embedding_type
         __UpperCamelCase    							=cross_attention_frequency
         __UpperCamelCase    							=encoder_hidden_size






    @classmethod
    def _a					(			cls					,   A_					,   **A_    )  ->       "PretrainedConfig":
         cls._set_token_in_kwargs(A_    )

         __UpperCamelCase							,      __UpperCamelCase    							=cls.get_config_dict(A_					,   **A_    )

         # get the qformer config dict if we are loading from InstructBlipConfig
         if config_dict.get('model_type'    ) == "instructblip":
              __UpperCamelCase    							=config_dict['qformer_config']

         if "model_type" in config_dict and hasattr(cls					,   'model_type'    ) and config_dict["model_type"] != cls.model_type:
              logger.warning(
                  f'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
                  f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'    )

         return cls.from_dict(A_					,   **A_    )






class     UpperCAmelCase__       (					A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Optional[int]      =  "instructblip"
    UpperCAmelCase__						:		Optional[Any]      =  True






    def __init__(			self					,   A_=None					,   A_=None					,   A_=None					,   A_=32					,   **A_    )  ->       List[str]:
         super().__init__(**A_    )

         if vision_config is None:
              __UpperCamelCase    							={}
              logger.info('vision_config is None. initializing the InstructBlipVisionConfig with default values.'    )

         if qformer_config is None:
              __UpperCamelCase    							={}
              logger.info('qformer_config is None. Initializing the InstructBlipQFormerConfig with default values.'    )

         if text_config is None:
              __UpperCamelCase    							={}
              logger.info('text_config is None. Initializing the text config with default values (`OPTConfig`).'    )

         __UpperCamelCase    							=InstructBlipVisionConfig(**A_    )
         __UpperCamelCase    							=InstructBlipQFormerConfig(**A_    )
         __UpperCamelCase    							=text_config['model_type'] if 'model_type' in text_config else 'opt'
         __UpperCamelCase    							=CONFIG_MAPPING[text_model_type](**A_    )

         __UpperCamelCase    							=self.text_config.tie_word_embeddings
         __UpperCamelCase    							=self.text_config.is_encoder_decoder

         __UpperCamelCase    							=num_query_tokens
         __UpperCamelCase    							=self.vision_config.hidden_size
         __UpperCamelCase    							=self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
         __UpperCamelCase    							=1.0
         __UpperCamelCase    							=0.02






    @classmethod
    def _a					(			cls					,   A_					,   A_					,   A_					,   **A_					,   )  ->       Optional[Any]:
         return cls(
             vision_config=vision_config.to_dict()					,   qformer_config=qformer_config.to_dict()					,   text_config=text_config.to_dict()					,   **A_					,   )






    def _a					(			self    )  ->       Optional[Any]:
         __UpperCamelCase    							=copy.deepcopy(self.__dict__    )
         __UpperCamelCase    							=self.vision_config.to_dict()
         __UpperCamelCase    							=self.qformer_config.to_dict()
         __UpperCamelCase    							=self.text_config.to_dict()
         __UpperCamelCase    							=self.__class__.model_type
         return output





