





'''simple docstring'''






import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				_A :				int				=  tmp_path / '''text.txt'''
				_A :				int				=  textwrap.dedent(
				    '''\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        ''')
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Tuple):



				"""simple docstring"""


				_A :				str				=  tmp_path / '''text_with_image.txt'''
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.mark.parametrize('''keep_linebreaks'''   , [True, False])
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read().splitlines(keepends=lowerCAmelCase)
				_A :				Any				=  Text(keep_linebreaks=lowerCAmelCase   , encoding='''utf-8''')
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				Dict				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content





@require_pil
def      lowercase  (  lowerCAmelCase :						Dict):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Tuple				=  f.read().splitlines()[0]
				_A :				Optional[int]				=  Text(encoding='''utf-8'''   , features=Features({'''image''': Image()}))
				_A :				Any				=  text._generate_tables([[text_file_with_image]])
				_A :				str				=  pa.concat_tables([table for _, table in generator])
				assert pa_table.schema.field('''image''').type == Image()()
				_A :				List[str]				=  pa_table.to_pydict()['''image''']
				assert generated_content == [{"path": image_file, "bytes": None}]





@pytest.mark.parametrize('''sample_by'''   , ['''line''', '''paragraph''', '''document'''])
def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						Optional[int]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read()
				if sample_by == "line":
								_A :				str				=  expected_content.splitlines()
				elif sample_by == "paragraph":
								_A :				str				=  expected_content.split('''\n\n''')
				elif sample_by == "document":
								_A :				Tuple				=  [expected_content]
				_A :				str				=  Text(sample_by=lowerCAmelCase   , encoding='''utf-8'''   , chunksize=100)
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				int				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content







'''simple docstring'''






import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__UpperCamelCase      :   List[Any]								=     get_logger(__name__)


if TYPE_CHECKING:
	from .builder import DatasetBuilder





def      lowercase  (  lowerCAmelCase :						List[str]   , lowerCAmelCase :						Optional[Union[str, bool]] = None):



				"""simple docstring"""


				_A :				Union[str, Any]				=  importlib.import_module(lowerCAmelCase)

				# TODO(QL): always update the module to add subsequent new authentication
				if hasattr(lowerCAmelCase   , '''_patched_for_streaming''') and module._patched_for_streaming:
								return

				def wrap_auth(lowerCAmelCase :						Union[str, Any]):
								@wraps(lowerCAmelCase)
								def wrapper(*lowerCAmelCase :						Union[str, Any]   , **lowerCAmelCase :						int):
												return function(*lowerCAmelCase   , use_auth_token=lowerCAmelCase   , **lowerCAmelCase)

								_A :				Optional[Any]				=  '''wrap_auth'''
								return wrapper

				# open files in a streaming fashion
				patch_submodule(lowerCAmelCase   , '''open'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.listdir'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.walk'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''glob.glob'''   , wrap_auth(lowerCAmelCase)).start()
				# allow to navigate in remote zip files
				patch_submodule(lowerCAmelCase   , '''os.path.join'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.dirname'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.basename'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.relpath'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.split'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.splitext'''   , lowerCAmelCase).start()
				# allow checks on paths
				patch_submodule(lowerCAmelCase   , '''os.path.exists'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.isdir'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.isfile'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.getsize'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''pathlib.Path'''   , lowerCAmelCase).start()
				# file readers
				patch_submodule(lowerCAmelCase   , '''gzip.open'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''numpy.load'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''pandas.read_csv'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''pandas.read_excel'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''scipy.io.loadmat'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''xml.etree.ElementTree.parse'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''xml.dom.minidom.parse'''   , wrap_auth(lowerCAmelCase)).start()
				_A :				Optional[int]				=  True





def      lowercase  (  lowerCAmelCase :						"DatasetBuilder"):



				"""simple docstring"""


				extend_module_for_streaming(builder.__module__   , use_auth_token=builder.use_auth_token)
				# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
				if not builder.__module__.startswith('''datasets.'''):  # check that it's not a packaged builder like csv
								for imports in get_imports(inspect.getfile(builder.__class__)):
												if imports[0] == "internal":
																_A :				Optional[int]				=  imports[1]
																_A :				str				=  '''.'''.join(builder.__module__.split('''.''')[:-1] + [internal_import_name])
																extend_module_for_streaming(lowerCAmelCase   , use_auth_token=builder.use_auth_token)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
				from .builder import DatasetBuilder

				_A :				Tuple				=  [
				    cls.__module__
				    for cls in type(lowerCAmelCase).__mro__[1:]  # make sure it's not the same module we've already patched
				    if issubclass(cls   , lowerCAmelCase) and cls.__module__ != DatasetBuilder.__module__
				]  # check it's not a standard builder from datasets.builder
				for module in parent_builder_modules:
								extend_module_for_streaming(lowerCAmelCase   , use_auth_token=builder.use_auth_token)
