






import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
__UpperCAmelCase								=						'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
__UpperCAmelCase								=						direct_transformers_import(PATH_TO_TRANSFORMERS)

__UpperCAmelCase								=						transformers.models.auto.configuration_auto.CONFIG_MAPPING

__UpperCAmelCase								=						{
    # used to compute the property `self.chunk_length`
    'EncodecConfig': ['overlap'],
    # used as `self.bert_model = BertModel(config, ...)`
    'DPRConfig': True,
    # not used in modeling files, but it's an important information
    'FSMTConfig': ['langs'],
    # used internally in the configuration class file
    'GPTNeoConfig': ['attention_types'],
    # used internally in the configuration class file
    'EsmConfig': ['is_folding_model'],
    # used during training (despite we don't have training script for these models yet)
    'Mask2FormerConfig': ['ignore_value'],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    'OneFormerConfig': ['ignore_value', 'norm'],
    # used during preprocessing and collation, see `collating_graphormer.py`
    'GraphormerConfig': ['spatial_pos_max'],
    # used internally in the configuration class file
    'T5Config': ['feed_forward_proj'],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    'MT5Config': ['feed_forward_proj', 'tokenizer_class'],
    'UMT5Config': ['feed_forward_proj', 'tokenizer_class'],
    # used internally in the configuration class file
    'LongT5Config': ['feed_forward_proj'],
    # used internally in the configuration class file
    'SwitchTransformersConfig': ['feed_forward_proj'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'BioGptConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'GLPNConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'SegformerConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'CvtConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'PerceiverConfig': ['layer_norm_eps'],
    # used internally to calculate the feature size
    'InformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate the feature size
    'TimeSeriesTransformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate the feature size
    'AutoformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate `mlp_dim`
    'SamVisionConfig': ['mlp_ratio'],
    # For (head) training, but so far not implemented
    'ClapAudioConfig': ['num_classes'],
    # Not used, but providing useful information to users
    'SpeechT5HifiGanConfig': ['sampling_rate'],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        'CLIPSegConfig': True,
        'DeformableDetrConfig': True,
        'DetaConfig': True,
        'DinatConfig': True,
        'DonutSwinConfig': True,
        'EfficientFormerConfig': True,
        'FSMTConfig': True,
        'JukeboxConfig': True,
        'LayoutLMv2Config': True,
        'MaskFormerSwinConfig': True,
        'MT5Config': True,
        'NatConfig': True,
        'OneFormerConfig': True,
        'PerceiverConfig': True,
        'RagConfig': True,
        'SpeechT5Config': True,
        'SwinConfig': True,
        'Swin2SRConfig': True,
        'Swinv2Config': True,
        'SwitchTransformersConfig': True,
        'TableTransformerConfig': True,
        'TapasConfig': True,
        'TransfoXLConfig': True,
        'UniSpeechConfig': True,
        'UniSpeechSatConfig': True,
        'WavLMConfig': True,
        'WhisperConfig': True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        'JukeboxPriorConfig': True,
        # TODO: @Younes (for `is_decoder`)
        'Pix2StructTextConfig': True,
    }
)




def      lowercase__      (     __snake_case    :							Optional[int]     ,       __snake_case    :							int     ,       __snake_case    :							Optional[Any]     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						False
					for attribute in attributes:
										for modeling_source in source_strings:
															# check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
															if (
															    F"config.{attribute}" in modeling_source
															    or F"getattr(config, \"{attribute}\"" in modeling_source
															    or F"getattr(self.config, \"{attribute}\"" in modeling_source
															):
																				UpperCAmelCase_    :      Dict            =						True
															# Deal with multi-line cases
															elif (
															    re.search(
															        RF"getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*\"{attribute}\""     ,       __snake_case     ,       )
															    is not None
															):
																				UpperCAmelCase_    :      Optional[int]            =						True
															# `SequenceSummary` is called with `SequenceSummary(config)`
															elif attribute in [
															    "summary_type",
															    "summary_use_proj",
															    "summary_activation",
															    "summary_last_dropout",
															    "summary_proj_to_labels",
															    "summary_first_dropout",
															]:
																				if "SequenceSummary" in modeling_source:
																									UpperCAmelCase_    :      List[Any]            =						True
															if attribute_used:
																				break
										if attribute_used:
															break

    # common and important attributes, even if they do not always appear in the modeling files
					UpperCAmelCase_    :      Union[str, Any]            =						[
					    'bos_index',
					    'eos_index',
					    'pad_index',
					    'unk_index',
					    'mask_index',
					    'image_size',
					    'use_cache',
					    'out_features',
					    'out_indices',
					]
					UpperCAmelCase_    :      Optional[int]            =						['encoder_no_repeat_ngram_size']

					# Special cases to be allowed
					UpperCAmelCase_    :      Union[str, Any]            =						True
					if not attribute_used:
										UpperCAmelCase_    :      List[str]            =						False
										for attribute in attributes:
															# Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
															if attribute in ["is_encoder_decoder"] and default_value is True:
																				UpperCAmelCase_    :      Any            =						True
															elif attribute in ["tie_word_embeddings"] and default_value is False:
																				UpperCAmelCase_    :      List[str]            =						True

															# Allow cases without checking the default value in the configuration class
															elif attribute in attributes_to_allow + attributes_used_in_generation:
																				UpperCAmelCase_    :      List[Any]            =						True
															elif attribute.endswith('_token_id'  ):
																				UpperCAmelCase_    :      Optional[int]            =						True

															# configuration class specific cases
															if not case_allowed:
																				UpperCAmelCase_    :      List[str]            =						SPECIAL_CASES_TO_ALLOW.get(config_class.__name__     ,       []  )
																				UpperCAmelCase_    :      Union[str, Any]            =						allowed_cases is True or attribute in allowed_cases

					return attribute_used or case_allowed




def      lowercase__      (     __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						dict(inspect.signature(config_class.__init__  ).parameters  )
					UpperCAmelCase_    :      str            =						[x for x in list(signature.keys()  ) if x not in ['self', 'kwargs']]
					UpperCAmelCase_    :      Dict            =						[signature[param].default for param in parameter_names]

					# If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
					# as one variant is used, the test should pass
					UpperCAmelCase_    :      Union[str, Any]            =						{}
					if len(config_class.attribute_map  ) > 0:
										UpperCAmelCase_    :      Tuple            =						{v: k for k, v in config_class.attribute_map.items()}

					# Get the path to modeling source files
					UpperCAmelCase_    :      Any            =						inspect.getsourcefile(__snake_case  )
					UpperCAmelCase_    :      Optional[int]            =						os.path.dirname(__snake_case  )
					# Let's check against all frameworks: as long as one framework uses an attribute, we are good.
					UpperCAmelCase_    :      Any            =						[os.path.join(__snake_case     ,       __snake_case  ) for fn in os.listdir(__snake_case  ) if fn.startswith('modeling_'  )]

					# Get the source code strings
					UpperCAmelCase_    :      Any            =						[]
					for path in modeling_paths:
										if os.path.isfile(__snake_case  ):
															with open(__snake_case  ) as fp:
																				modeling_sources.append(fp.read()  )

					UpperCAmelCase_    :      Any            =						[]
					for config_param, default_value in zip(__snake_case     ,       __snake_case  ):
										# `attributes` here is all the variant names for `config_param`
										UpperCAmelCase_    :      Dict            =						[config_param]
										# some configuration classes have non-empty `attribute_map`, and both names could be used in the
										# corresponding modeling files. As long as one of them appears, it is fine.
										if config_param in reversed_attribute_map:
															attributes.append(reversed_attribute_map[config_param]  )

										if not check_attribute_being_used(__snake_case     ,       __snake_case     ,       __snake_case     ,       __snake_case  ):
															unused_attributes.append(attributes[0]  )

					return sorted(__snake_case  )




def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      Any            =						{}
					for _config_class in list(CONFIG_MAPPING.values()  ):
										# Skip deprecated models
										if "models.deprecated" in _config_class.__module__:
															continue
										# Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
										UpperCAmelCase_    :      Dict            =						[
										    cls
										    for name, cls in inspect.getmembers(
										        inspect.getmodule(_config_class  )     ,       lambda __snake_case  : inspect.isclass(__snake_case  )
										        and issubclass(__snake_case     ,       __snake_case  )
										        and inspect.getmodule(__snake_case  ) == inspect.getmodule(_config_class  )     ,       )
										]
										for config_class in config_classes_in_module:
															UpperCAmelCase_    :      int            =						check_config_attributes_being_used(__snake_case  )
															if len(__snake_case  ) > 0:
																				UpperCAmelCase_    :      Optional[Any]            =						unused_attributes

					if len(__snake_case  ) > 0:
										UpperCAmelCase_    :      int            =						'The following configuration classes contain unused attributes in the corresponding modeling files:\n'
										for name, attributes in configs_with_unused_attributes.items():
															error += F"{name}: {attributes}\n"

										raise ValueError(__snake_case  )


if __name__ == "__main__":
		check_config_attributes()














import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						10
					UpperCAmelCase_    :      Tuple            =						datasets.Features(
					    {
					        'tokens': datasets.Sequence(datasets.Value('string'  )  ),
					        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']  )  ),
					        'answers': datasets.Sequence(
					            {
					                'text': datasets.Value('string'  ),
					                'answer_start': datasets.Value('int32'  ),
					            }  ),
					        'id': datasets.Value('int64'  ),
					    }  )
					UpperCAmelCase_    :      Tuple            =						datasets.Dataset.from_dict(
					    {
					        'tokens': [['foo'] * 5] * n,
					        'labels': [[1] * 5] * n,
					        'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
					        'id': list(range(__snake_case  )  ),
					    }     ,       features=__snake_case     ,       )
					return dataset




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							List[str]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						str(tmp_path_factory.mktemp('data'  ) / 'file.arrow'  )
					dataset.map(cache_file_name=__snake_case  )
					return filename


# FILE_CONTENT + files


__UpperCAmelCase								=						'\\n    Text data.\n    Second line of data.'




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt'
					UpperCAmelCase_    :      Tuple            =						FILE_CONTENT
					with open(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case  )
					return filename




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]  ):


					'''simple docstring'''


					import bza

					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.bz2'
					UpperCAmelCase_    :      str            =						bytes(__snake_case     ,       'utf-8'  )
					with bza.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'file.txt.gz'  )
					UpperCAmelCase_    :      Dict            =						bytes(__snake_case     ,       'utf-8'  )
					with gzip.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if datasets.config.LZ4_AVAILABLE:
										import lza.frame

										UpperCAmelCase_    :      Any            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.lz4'
										UpperCAmelCase_    :      Any            =						bytes(__snake_case     ,       'utf-8'  )
										with lza.frame.open(__snake_case     ,       'wb'  ) as f:
															f.write(__snake_case  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					if datasets.config.PY7ZR_AVAILABLE:
										import pyazr

										UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.7z'
										with pyazr.SevenZipFile(__snake_case     ,       'w'  ) as archive:
															archive.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					import tarfile

					UpperCAmelCase_    :      Any            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					import lzma

					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.xz'
					UpperCAmelCase_    :      Any            =						bytes(__snake_case     ,       'utf-8'  )
					with lzma.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[int]     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					import zipfile

					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					if datasets.config.ZSTANDARD_AVAILABLE:
										import zstandard as zstd

										UpperCAmelCase_    :      Tuple            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.zst'
										UpperCAmelCase_    :      List[str]            =						bytes(__snake_case     ,       'utf-8'  )
										with zstd.open(__snake_case     ,       'wb'  ) as f:
															f.write(__snake_case  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.xml'
					UpperCAmelCase_    :      List[Any]            =						textwrap.dedent(
					    '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'  )
					with open(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case  )
					return filename


__UpperCAmelCase								=						[
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
__UpperCAmelCase								=						[
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
__UpperCAmelCase								=						{
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

__UpperCAmelCase								=						[
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

__UpperCAmelCase								=						[
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return DATA_DICT_OF_LISTS




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						datasets.Dataset.from_dict(__snake_case  )
					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.arrow'  )
					dataset.map(cache_file_name=__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.sqlite'  )
					with contextlib.closing(sqlitea.connect(__snake_case  )  ) as con:
										UpperCAmelCase_    :      List[Any]            =						con.cursor()
										cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'  )
										for item in DATA:
															cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'     ,       tuple(item.values()  )  )
										con.commit()
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.csv'  )
					with open(__snake_case     ,       'w'     ,       newline=''  ) as f:
										UpperCAmelCase_    :      Tuple            =						csv.DictWriter(__snake_case     ,       fieldnames=['col_1', 'col_2', 'col_3']  )
										writer.writeheader()
										for item in DATA:
															writer.writerow(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.csv'  )
					with open(__snake_case     ,       'w'     ,       newline=''  ) as f:
										UpperCAmelCase_    :      Optional[Any]            =						csv.DictWriter(__snake_case     ,       fieldnames=['col_1', 'col_2', 'col_3']  )
										writer.writeheader()
										for item in DATA:
															writer.writerow(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					import bza

					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.bz2'
					with open(__snake_case     ,       'rb'  ) as f:
										UpperCAmelCase_    :      int            =						f.read()
					# data = bytes(FILE_CONTENT, "utf-8")
					with bza.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							Tuple     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Optional[int]     ,       __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(csv_path.replace('.csv'     ,       '.CSV'  )  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(csva_path.replace('.csv'     ,       '.CSV'  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							int     ,       __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.parquet'  )
					UpperCAmelCase_    :      Dict            =						pa.schema(
					    {
					        'col_1': pa.string(),
					        'col_2': pa.intaa(),
					        'col_3': pa.floataa(),
					    }  )
					with open(__snake_case     ,       'wb'  ) as f:
										UpperCAmelCase_    :      List[Any]            =						pq.ParquetWriter(__snake_case     ,       schema=__snake_case  )
										UpperCAmelCase_    :      Any            =						pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__snake_case  )  )] for k in DATA[0]}     ,       schema=__snake_case  )
										writer.write_table(__snake_case  )
										writer.close()
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
					UpperCAmelCase_    :      Optional[int]            =						{'data': DATA}
					with open(__snake_case     ,       'w'  ) as f:
										json.dump(__snake_case     ,       __snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
					UpperCAmelCase_    :      Tuple            =						{'data': DATA_DICT_OF_LISTS}
					with open(__snake_case     ,       'w'  ) as f:
										json.dump(__snake_case     ,       __snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset_312.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA_312:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset-str.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA_STR:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Union[str, Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt.gz'  )
					with open(__snake_case     ,       'rb'  ) as orig_file:
										with gzip.open(__snake_case     ,       'wb'  ) as zipped_file:
															zipped_file.writelines(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.gz'  )
					with open(__snake_case     ,       'rb'  ) as orig_file:
										with gzip.open(__snake_case     ,       'wb'  ) as zipped_file:
															zipped_file.writelines(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							Dict     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							str     ,       __snake_case    :							Dict     ,       __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('nested'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Union[str, Any]     ,       __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							str     ,       __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Any     ,       __snake_case    :							Any     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.join('nested'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Any            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.txt'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      List[str]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.abc'
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any     ,       __snake_case    :							Union[str, Any]     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.text.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							str     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.text.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]     ,       __snake_case    :							str     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.ext.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename('unsupported.ext'  )  )
										f.write(__snake_case     ,       arcname=os.path.basename('unsupported_2.ext'  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						'\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']  )
					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset_with_unicode_new_lines.txt'  )
					with open(__snake_case     ,       'w'     ,       encoding='utf-8'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return os.path.join('tests'     ,       'features'     ,       'data'     ,       'test_image_rgb.jpg'  )




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return os.path.join('tests'     ,       'features'     ,       'data'     ,       'test_audio_44100.wav'  )




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							List[str]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.img.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  ).replace('.jpg'     ,       '2.jpg'  )  )
					return path






@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data_dir'  )

					(data_dir / "subdir").mkdir()
					with open(data_dir / 'subdir' / 'train.txt'     ,       'w'  ) as f:
										f.write('foo\n' * 10  )
					with open(data_dir / 'subdir' / 'test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )
					# hidden file
					with open(data_dir / 'subdir' / '.test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )

					# hidden directory
					(data_dir / ".subdir").mkdir()
					with open(data_dir / '.subdir' / 'train.txt'     ,       'w'  ) as f:
										f.write('foo\n' * 10  )
					with open(data_dir / '.subdir' / 'test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )

					return data_dir






