



'''simple docstring'''







import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json
@pytest.fixture
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Union[str, Any]						) ->							str:
							lowerCAmelCase__  =   tmp_path / "file.jsonl"
							lowerCAmelCase__  =   textwrap.dedent(
							    """\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        """						)
							with open(_lowerCAmelCase					, """w"""						) as f:
														f.write(_lowerCAmelCase						)
							return str(_lowerCAmelCase						)
@pytest.fixture
def 			_lowerCAmelCase(      UpperCAmelCase_      :	int						) ->							Tuple:
							lowerCAmelCase__  =   tmp_path / "file_utf16_encoded.jsonl"
							lowerCAmelCase__  =   textwrap.dedent(
							    """\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        """						)
							with open(_lowerCAmelCase					, """w"""					, encoding="""utf-16"""						) as f:
														f.write(_lowerCAmelCase						)
							return str(_lowerCAmelCase						)
@pytest.fixture
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]						) ->							int:
							lowerCAmelCase__  =   tmp_path / "file_with_list_of_dicts.json"
							lowerCAmelCase__  =   textwrap.dedent(
							    """\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        """						)
							with open(_lowerCAmelCase					, """w"""						) as f:
														f.write(_lowerCAmelCase						)
							return str(_lowerCAmelCase						)
@pytest.fixture
def 			_lowerCAmelCase(      UpperCAmelCase_      :	List[str]						) ->							Optional[int]:
							lowerCAmelCase__  =   tmp_path / "file_with_list_of_dicts_field.json"
							lowerCAmelCase__  =   textwrap.dedent(
							    """\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        """						)
							with open(_lowerCAmelCase					, """w"""						) as f:
														f.write(_lowerCAmelCase						)
							return str(_lowerCAmelCase						)
@pytest.mark.parametrize(
    """file_fixture, config_kwargs"""					, [
        ("""jsonl_file""", {}),
        ("""jsonl_file_utf16_encoded""", {"""encoding""": """utf-16"""}),
        ("""json_file_with_list_of_dicts""", {}),
        ("""json_file_with_list_of_dicts_field""", {"""field""": """field3"""}),
    ]					, )
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Any					, UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	List[Any]						) ->							Any:
							lowerCAmelCase__  =   Json(**_lowerCAmelCase						)
							lowerCAmelCase__  =   json._generate_tables([[request.getfixturevalue(_lowerCAmelCase						)]]						)
							lowerCAmelCase__  =   pa.concat_tables([table for _, table in generator]						)
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}






@pytest.mark.parametrize(
    """file_fixture, config_kwargs"""					, [
        (
            """jsonl_file""",
            {"""features""": Features({"""col_1""": Value("""int64"""						), """col_2""": Value("""int64"""						), """missing_col""": Value("""string"""						)}						)},
        ),
        (
            """json_file_with_list_of_dicts""",
            {"""features""": Features({"""col_1""": Value("""int64"""						), """col_2""": Value("""int64"""						), """missing_col""": Value("""string"""						)}						)},
        ),
        (
            """json_file_with_list_of_dicts_field""",
            {
                """field""": """field3""",
                """features""": Features(
                    {"""col_1""": Value("""int64"""						), """col_2""": Value("""int64"""						), """missing_col""": Value("""string"""						)}						),
            },
        ),
    ]					, )
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]					, UpperCAmelCase_      :	List[str]					, UpperCAmelCase_      :	Any						) ->							Optional[Any]:
							lowerCAmelCase__  =   Json(**_lowerCAmelCase						)
							lowerCAmelCase__  =   json._generate_tables([[request.getfixturevalue(_lowerCAmelCase						)]]						)
							lowerCAmelCase__  =   pa.concat_tables([table for _, table in generator]						)
							assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}






'''simple docstring'''

import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 		SCREAMING_SNAKE_CASE_					(_UpperCAmelCase      ):



			"""simple docstring"""






			__magic_name__ :				Dict   							=       ["""image_processor""", """tokenizer"""]
			__magic_name__ :				Union[str, Any]   							=       """VideoMAEImageProcessor"""
			__magic_name__ :				List[str]   							=       ("""CLIPTokenizer""", """CLIPTokenizerFast""")



			def __init__(							self				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):
									snake_case :       int				=    None
									if "feature_extractor" in kwargs:
															warnings.warn(
															    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
															    " instead."				,			snake_case_				,			)
															snake_case :       Optional[int]				=    kwargs.pop("feature_extractor"					)

									snake_case :       Union[str, Any]				=    image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
															raise ValueError("You need to specify an `image_processor`."					)
									if tokenizer is None:
															raise ValueError("You need to specify a `tokenizer`."					)

									super().__init__(snake_case_				,			snake_case_					)
									snake_case :       List[Any]				=    self.image_processor



			def __call__(							self				,			snake_case_=None				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):

									if text is None and videos is None:
															raise ValueError("You have to specify either text or videos. Both cannot be none."					)

									if text is not None:
															snake_case :       List[Any]				=    self.tokenizer(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if videos is not None:
															snake_case :       List[Any]				=    self.image_processor(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if text is not None and videos is not None:
															snake_case :       Tuple				=    image_features.pixel_values
															return encoding
									elif text is not None:
															return encoding
									else:
															return BatchEncoding(data=dict(**snake_case_					)				,			tensor_type=snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.batch_decode(*snake_case_				,			**snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.decode(*snake_case_				,			**snake_case_					)



			@property
			def   lowerCamelCase (							self					):
									return ["input_ids", "attention_mask", "position_ids", "pixel_values"]



			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."				,			snake_case_				,			)
									return self.image_processor_class






			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."				,			snake_case_				,			)
									return self.image_processor





