






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase								=						logging.get_logger(__name__)


__UpperCAmelCase								=						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

__UpperCAmelCase								=						{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

__UpperCAmelCase								=						{
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						(
					    list(range(ord('!'  )     ,       ord('~'  ) + 1  )  ) + list(range(ord('¡'  )     ,       ord('¬'  ) + 1  )  ) + list(range(ord('®'  )     ,       ord('ÿ'  ) + 1  )  )
					)
					UpperCAmelCase_    :      Any            =						bs[:]
					UpperCAmelCase_    :      Tuple            =						0
					for b in range(2**8  ):
										if b not in bs:
															bs.append(__snake_case  )
															cs.append(2**8 + n  )
															n += 1
					UpperCAmelCase_    :      Optional[Any]            =						[chr(__snake_case  ) for n in cs]
					return dict(zip(__snake_case     ,       __snake_case  )  )




def      lowercase__      (     __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						set()
					UpperCAmelCase_    :      List[str]            =						word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)  )
										UpperCAmelCase_    :      Optional[Any]            =						char
					return pairs






class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		_snake_case    :    Optional[Any]       						=  VOCAB_FILES_NAMES
		_snake_case    :    Optional[int]       						=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case    :    List[str]       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case    :    str       						=  ['''input_ids''', '''attention_mask''']



		def __init__(							self				,		_UpperCamelCase				,		_UpperCamelCase				,		_UpperCamelCase="replace"				,		_UpperCamelCase="<s>"				,		_UpperCamelCase="</s>"				,		_UpperCamelCase="</s>"				,		_UpperCamelCase="<s>"				,		_UpperCamelCase="<unk>"				,		_UpperCamelCase="<pad>"				,		_UpperCamelCase="<mask>"				,		_UpperCamelCase=False				,		**_UpperCamelCase				,		)	->							Optional[int]:
							UpperCAmelCase_    :      Tuple            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else bos_token
							UpperCAmelCase_    :      List[str]            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else eos_token
							UpperCAmelCase_    :      Optional[int]            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else sep_token
							UpperCAmelCase_    :      int            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else cls_token
							UpperCAmelCase_    :      Dict            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else unk_token
							UpperCAmelCase_    :      Optional[int]            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_    :      Dict            =						AddedToken(_UpperCamelCase				,		lstrip=_UpperCamelCase				,		rstrip=_UpperCamelCase   ) if isinstance(_UpperCamelCase				,		_UpperCamelCase   ) else mask_token

							super().__init__(
							    errors=_UpperCamelCase				,		bos_token=_UpperCamelCase				,		eos_token=_UpperCamelCase				,		unk_token=_UpperCamelCase				,		sep_token=_UpperCamelCase				,		cls_token=_UpperCamelCase				,		pad_token=_UpperCamelCase				,		mask_token=_UpperCamelCase				,		add_prefix_space=_UpperCamelCase				,		**_UpperCamelCase				,		)

							with open(_UpperCamelCase				,		encoding='utf-8'   ) as vocab_handle:
												UpperCAmelCase_    :      Any            =						json.load(_UpperCamelCase   )
							UpperCAmelCase_    :      Any            =						{v: k for k, v in self.encoder.items()}
							UpperCAmelCase_    :      List[Any]            =						errors  # how to handle errors in decoding
							UpperCAmelCase_    :      Any            =						bytes_to_unicode()
							UpperCAmelCase_    :      List[str]            =						{v: k for k, v in self.byte_encoder.items()}
							with open(_UpperCamelCase				,		encoding='utf-8'   ) as merges_handle:
												UpperCAmelCase_    :      int            =						merges_handle.read().split('\n'   )[1:-1]
							UpperCAmelCase_    :      Union[str, Any]            =						[tuple(merge.split()   ) for merge in bpe_merges]
							UpperCAmelCase_    :      List[str]            =						dict(zip(_UpperCamelCase				,		range(len(_UpperCamelCase   )   )   )   )
							UpperCAmelCase_    :      int            =						{}
							UpperCAmelCase_    :      Union[str, Any]            =						add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCAmelCase_    :      str            =						re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'   )



		@property
		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							return len(self.encoder   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							return dict(self.encoder				,		**self.added_tokens_encoder   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Any:
							if token in self.cache:
												return self.cache[token]
							UpperCAmelCase_    :      List[Any]            =						tuple(_UpperCamelCase   )
							UpperCAmelCase_    :      Optional[int]            =						get_pairs(_UpperCamelCase   )

							if not pairs:
												return token

							while True:
												UpperCAmelCase_    :      List[str]            =						min(_UpperCamelCase				,		key=lambda _UpperCamelCase   : self.bpe_ranks.get(_UpperCamelCase				,		float('inf'   )   )   )
												if bigram not in self.bpe_ranks:
																	break
												UpperCAmelCase_       ,  UpperCAmelCase_    :      Union[str, Any]            =						bigram
												UpperCAmelCase_    :      Dict            =						[]
												UpperCAmelCase_    :      Dict            =						0
												while i < len(_UpperCamelCase   ):
																	try:
																						UpperCAmelCase_    :      int            =						word.index(_UpperCamelCase				,		_UpperCamelCase   )
																	except ValueError:
																						new_word.extend(word[i:]   )
																						break
																	else:
																						new_word.extend(word[i:j]   )
																						UpperCAmelCase_    :      int            =						j

																	if word[i] == first and i < len(_UpperCamelCase   ) - 1 and word[i + 1] == second:
																						new_word.append(first + second   )
																						i += 2
																	else:
																						new_word.append(word[i]   )
																						i += 1
												UpperCAmelCase_    :      Tuple            =						tuple(_UpperCamelCase   )
												UpperCAmelCase_    :      Dict            =						new_word
												if len(_UpperCamelCase   ) == 1:
																	break
												else:
																	UpperCAmelCase_    :      Tuple            =						get_pairs(_UpperCamelCase   )
							UpperCAmelCase_    :      List[Any]            =						' '.join(_UpperCamelCase   )
							UpperCAmelCase_    :      List[str]            =						word
							return word



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Union[str, Any]:
							UpperCAmelCase_    :      List[str]            =						[]
							for token in re.findall(self.pat				,		_UpperCamelCase   ):
												UpperCAmelCase_    :      Dict            =						''.join(
												    self.byte_encoder[b] for b in token.encode('utf-8'   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCamelCase   ).split(' '   )   )
							return bpe_tokens



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							List[str]:
							return self.encoder.get(_UpperCamelCase				,		self.encoder.get(self.unk_token   )   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Any:
							return self.decoder.get(_UpperCamelCase   )



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Any:
							UpperCAmelCase_    :      Optional[Any]            =						''.join(_UpperCamelCase   )
							UpperCAmelCase_    :      str            =						bytearray([self.byte_decoder[c] for c in text]   ).decode('utf-8'				,		errors=self.errors   )
							return text



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase   ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"   )
												return
							UpperCAmelCase_    :      Tuple            =						os.path.join(
							    _UpperCamelCase				,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
							UpperCAmelCase_    :      Dict            =						os.path.join(
							    _UpperCamelCase				,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

							with open(_UpperCamelCase				,		'w'				,		encoding='utf-8'   ) as f:
												f.write(json.dumps(self.encoder				,		indent=2				,		sort_keys=_UpperCamelCase				,		ensure_ascii=_UpperCamelCase   ) + '\n'   )

							UpperCAmelCase_    :      Union[str, Any]            =						0
							with open(_UpperCamelCase				,		'w'				,		encoding='utf-8'   ) as writer:
												writer.write('#version: 0.2\n'   )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,		key=lambda _UpperCamelCase   : kv[1]   ):
																	if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																						    ' Please check that the tokenizer is not corrupted!'   )
																						UpperCAmelCase_    :      Union[str, Any]            =						token_index
																	writer.write(' '.join(_UpperCamelCase   ) + '\n'   )
																	index += 1

							return vocab_file, merge_file



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCAmelCase_    :      List[Any]            =						[self.cls_token_id]
							UpperCAmelCase_    :      Dict            =						[self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None				,		_UpperCamelCase = False   )	->							List[int]:
							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase				,		token_ids_a=_UpperCamelCase				,		already_has_special_tokens=_UpperCamelCase   )

							if token_ids_a is None:
												return [1] + ([0] * len(_UpperCamelCase   )) + [1]
							return [1] + ([0] * len(_UpperCamelCase   )) + [1, 1] + ([0] * len(_UpperCamelCase   )) + [1]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase = None   )	->							List[int]:
							UpperCAmelCase_    :      Dict            =						[self.sep_token_id]
							UpperCAmelCase_    :      str            =						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase				,		_UpperCamelCase=False				,		**_UpperCamelCase   )	->							Optional[Any]:
							UpperCAmelCase_    :      Optional[int]            =						kwargs.pop('add_prefix_space'				,		self.add_prefix_space   )
							if (is_split_into_words or add_prefix_space) and (len(_UpperCamelCase   ) > 0 and not text[0].isspace()):
												UpperCAmelCase_    :      Optional[Any]            =						' ' + text
							return (text, kwargs)














import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						10
					UpperCAmelCase_    :      Tuple            =						datasets.Features(
					    {
					        'tokens': datasets.Sequence(datasets.Value('string'  )  ),
					        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']  )  ),
					        'answers': datasets.Sequence(
					            {
					                'text': datasets.Value('string'  ),
					                'answer_start': datasets.Value('int32'  ),
					            }  ),
					        'id': datasets.Value('int64'  ),
					    }  )
					UpperCAmelCase_    :      Tuple            =						datasets.Dataset.from_dict(
					    {
					        'tokens': [['foo'] * 5] * n,
					        'labels': [[1] * 5] * n,
					        'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
					        'id': list(range(__snake_case  )  ),
					    }     ,       features=__snake_case     ,       )
					return dataset




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							List[str]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						str(tmp_path_factory.mktemp('data'  ) / 'file.arrow'  )
					dataset.map(cache_file_name=__snake_case  )
					return filename


# FILE_CONTENT + files


__UpperCAmelCase								=						'\\n    Text data.\n    Second line of data.'




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt'
					UpperCAmelCase_    :      Tuple            =						FILE_CONTENT
					with open(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case  )
					return filename




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]  ):


					'''simple docstring'''


					import bza

					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.bz2'
					UpperCAmelCase_    :      str            =						bytes(__snake_case     ,       'utf-8'  )
					with bza.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'file.txt.gz'  )
					UpperCAmelCase_    :      Dict            =						bytes(__snake_case     ,       'utf-8'  )
					with gzip.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if datasets.config.LZ4_AVAILABLE:
										import lza.frame

										UpperCAmelCase_    :      Any            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.lz4'
										UpperCAmelCase_    :      Any            =						bytes(__snake_case     ,       'utf-8'  )
										with lza.frame.open(__snake_case     ,       'wb'  ) as f:
															f.write(__snake_case  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					if datasets.config.PY7ZR_AVAILABLE:
										import pyazr

										UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.7z'
										with pyazr.SevenZipFile(__snake_case     ,       'w'  ) as archive:
															archive.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					import tarfile

					UpperCAmelCase_    :      Any            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					import lzma

					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.xz'
					UpperCAmelCase_    :      Any            =						bytes(__snake_case     ,       'utf-8'  )
					with lzma.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[int]     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					import zipfile

					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					if datasets.config.ZSTANDARD_AVAILABLE:
										import zstandard as zstd

										UpperCAmelCase_    :      Tuple            =						tmp_path_factory.mktemp('data'  ) / 'file.txt.zst'
										UpperCAmelCase_    :      List[str]            =						bytes(__snake_case     ,       'utf-8'  )
										with zstd.open(__snake_case     ,       'wb'  ) as f:
															f.write(__snake_case  )
										return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'file.xml'
					UpperCAmelCase_    :      List[Any]            =						textwrap.dedent(
					    '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'  )
					with open(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case  )
					return filename


__UpperCAmelCase								=						[
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
__UpperCAmelCase								=						[
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
__UpperCAmelCase								=						{
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

__UpperCAmelCase								=						[
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

__UpperCAmelCase								=						[
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return DATA_DICT_OF_LISTS




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						datasets.Dataset.from_dict(__snake_case  )
					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.arrow'  )
					dataset.map(cache_file_name=__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.sqlite'  )
					with contextlib.closing(sqlitea.connect(__snake_case  )  ) as con:
										UpperCAmelCase_    :      List[Any]            =						con.cursor()
										cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'  )
										for item in DATA:
															cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'     ,       tuple(item.values()  )  )
										con.commit()
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.csv'  )
					with open(__snake_case     ,       'w'     ,       newline=''  ) as f:
										UpperCAmelCase_    :      Tuple            =						csv.DictWriter(__snake_case     ,       fieldnames=['col_1', 'col_2', 'col_3']  )
										writer.writeheader()
										for item in DATA:
															writer.writerow(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.csv'  )
					with open(__snake_case     ,       'w'     ,       newline=''  ) as f:
										UpperCAmelCase_    :      Optional[Any]            =						csv.DictWriter(__snake_case     ,       fieldnames=['col_1', 'col_2', 'col_3']  )
										writer.writeheader()
										for item in DATA:
															writer.writerow(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					import bza

					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.bz2'
					with open(__snake_case     ,       'rb'  ) as f:
										UpperCAmelCase_    :      int            =						f.read()
					# data = bytes(FILE_CONTENT, "utf-8")
					with bza.open(__snake_case     ,       'wb'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							Tuple     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Optional[int]     ,       __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(csv_path.replace('.csv'     ,       '.CSV'  )  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(csva_path.replace('.csv'     ,       '.CSV'  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							int     ,       __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.csv.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.parquet'  )
					UpperCAmelCase_    :      Dict            =						pa.schema(
					    {
					        'col_1': pa.string(),
					        'col_2': pa.intaa(),
					        'col_3': pa.floataa(),
					    }  )
					with open(__snake_case     ,       'wb'  ) as f:
										UpperCAmelCase_    :      List[Any]            =						pq.ParquetWriter(__snake_case     ,       schema=__snake_case  )
										UpperCAmelCase_    :      Any            =						pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__snake_case  )  )] for k in DATA[0]}     ,       schema=__snake_case  )
										writer.write_table(__snake_case  )
										writer.close()
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
					UpperCAmelCase_    :      Optional[int]            =						{'data': DATA}
					with open(__snake_case     ,       'w'  ) as f:
										json.dump(__snake_case     ,       __snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.json'  )
					UpperCAmelCase_    :      Tuple            =						{'data': DATA_DICT_OF_LISTS}
					with open(__snake_case     ,       'w'  ) as f:
										json.dump(__snake_case     ,       __snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset_312.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA_312:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset-str.jsonl'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in DATA_STR:
															f.write(json.dumps(__snake_case  ) + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Union[str, Any]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt.gz'  )
					with open(__snake_case     ,       'rb'  ) as orig_file:
										with gzip.open(__snake_case     ,       'wb'  ) as zipped_file:
															zipped_file.writelines(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							int     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					import gzip

					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.gz'  )
					with open(__snake_case     ,       'rb'  ) as orig_file:
										with gzip.open(__snake_case     ,       'wb'  ) as zipped_file:
															zipped_file.writelines(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							Dict     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							str     ,       __snake_case    :							Dict     ,       __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('nested'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Union[str, Any]     ,       __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[int]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.jsonl.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							str     ,       __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.jsonl.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.add(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Any     ,       __snake_case    :							Any     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_nested.jsonl.tar'
					with tarfile.TarFile(__snake_case     ,       'w'  ) as f:
										f.add(__snake_case     ,       arcname=os.path.join('nested'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Any            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset.txt'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset2.txt'  )
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						['0', '1', '2', '3']
					UpperCAmelCase_    :      List[str]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.abc'
					with open(__snake_case     ,       'w'  ) as f:
										for item in data:
															f.write(item + '\n'  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any     ,       __snake_case    :							Union[str, Any]     ,       __snake_case    :							List[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.text.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							str     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset_with_dir.text.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
										f.write(__snake_case     ,       arcname=os.path.join('main_dir'     ,       os.path.basename(__snake_case  )  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Union[str, Any]     ,       __snake_case    :							str     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.ext.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename('unsupported.ext'  )  )
										f.write(__snake_case     ,       arcname=os.path.basename('unsupported_2.ext'  )  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Tuple            =						'\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']  )
					UpperCAmelCase_    :      Dict            =						str(tmp_path_factory.mktemp('data'  ) / 'dataset_with_unicode_new_lines.txt'  )
					with open(__snake_case     ,       'w'     ,       encoding='utf-8'  ) as f:
										f.write(__snake_case  )
					return path




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return os.path.join('tests'     ,       'features'     ,       'data'     ,       'test_image_rgb.jpg'  )




@pytest.fixture(scope='session'  )
def      lowercase__      (     ):


					'''simple docstring'''


					return os.path.join('tests'     ,       'features'     ,       'data'     ,       'test_audio_44100.wav'  )




@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							List[str]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data'  ) / 'dataset.img.zip'
					with zipfile.ZipFile(__snake_case     ,       'w'  ) as f:
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  )  )
										f.write(__snake_case     ,       arcname=os.path.basename(__snake_case  ).replace('.jpg'     ,       '2.jpg'  )  )
					return path






@pytest.fixture(scope='session'  )
def      lowercase__      (     __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						tmp_path_factory.mktemp('data_dir'  )

					(data_dir / "subdir").mkdir()
					with open(data_dir / 'subdir' / 'train.txt'     ,       'w'  ) as f:
										f.write('foo\n' * 10  )
					with open(data_dir / 'subdir' / 'test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )
					# hidden file
					with open(data_dir / 'subdir' / '.test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )

					# hidden directory
					(data_dir / ".subdir").mkdir()
					with open(data_dir / '.subdir' / 'train.txt'     ,       'w'  ) as f:
										f.write('foo\n' * 10  )
					with open(data_dir / '.subdir' / 'test.txt'     ,       'w'  ) as f:
										f.write('bar\n' * 10  )

					return data_dir






