





'''simple docstring'''






# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus

# 1. pip install sentencepiece
#
# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt

# 3. build
import sentencepiece as spm

# pegasus:
# 1. no bos
# 2. eos_id is 1
# 3. unk_id is 2
# build a sample spm file accordingly
spm.SentencePieceTrainer.train(
    '''--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000'''
)

# 4. now update the fixture
# mv test_sentencepiece_no_bos.model ../../tests/fixtures/







'''simple docstring'''






import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__UpperCamelCase      :   List[Any]								=     get_logger(__name__)


if TYPE_CHECKING:
	from .builder import DatasetBuilder





def      lowercase  (  lowerCAmelCase :						List[str]   , lowerCAmelCase :						Optional[Union[str, bool]] = None):



				"""simple docstring"""


				_A :				Union[str, Any]				=  importlib.import_module(lowerCAmelCase)

				# TODO(QL): always update the module to add subsequent new authentication
				if hasattr(lowerCAmelCase   , '''_patched_for_streaming''') and module._patched_for_streaming:
								return

				def wrap_auth(lowerCAmelCase :						Union[str, Any]):
								@wraps(lowerCAmelCase)
								def wrapper(*lowerCAmelCase :						Union[str, Any]   , **lowerCAmelCase :						int):
												return function(*lowerCAmelCase   , use_auth_token=lowerCAmelCase   , **lowerCAmelCase)

								_A :				Optional[Any]				=  '''wrap_auth'''
								return wrapper

				# open files in a streaming fashion
				patch_submodule(lowerCAmelCase   , '''open'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.listdir'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.walk'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''glob.glob'''   , wrap_auth(lowerCAmelCase)).start()
				# allow to navigate in remote zip files
				patch_submodule(lowerCAmelCase   , '''os.path.join'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.dirname'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.basename'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.relpath'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.split'''   , lowerCAmelCase).start()
				patch_submodule(lowerCAmelCase   , '''os.path.splitext'''   , lowerCAmelCase).start()
				# allow checks on paths
				patch_submodule(lowerCAmelCase   , '''os.path.exists'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.isdir'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.isfile'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''os.path.getsize'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''pathlib.Path'''   , lowerCAmelCase).start()
				# file readers
				patch_submodule(lowerCAmelCase   , '''gzip.open'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''numpy.load'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''pandas.read_csv'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''pandas.read_excel'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''scipy.io.loadmat'''   , wrap_auth(lowerCAmelCase)   , attrs=['''__version__''']).start()
				patch_submodule(lowerCAmelCase   , '''xml.etree.ElementTree.parse'''   , wrap_auth(lowerCAmelCase)).start()
				patch_submodule(lowerCAmelCase   , '''xml.dom.minidom.parse'''   , wrap_auth(lowerCAmelCase)).start()
				_A :				Optional[int]				=  True





def      lowercase  (  lowerCAmelCase :						"DatasetBuilder"):



				"""simple docstring"""


				extend_module_for_streaming(builder.__module__   , use_auth_token=builder.use_auth_token)
				# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
				if not builder.__module__.startswith('''datasets.'''):  # check that it's not a packaged builder like csv
								for imports in get_imports(inspect.getfile(builder.__class__)):
												if imports[0] == "internal":
																_A :				Optional[int]				=  imports[1]
																_A :				str				=  '''.'''.join(builder.__module__.split('''.''')[:-1] + [internal_import_name])
																extend_module_for_streaming(lowerCAmelCase   , use_auth_token=builder.use_auth_token)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
				from .builder import DatasetBuilder

				_A :				Tuple				=  [
				    cls.__module__
				    for cls in type(lowerCAmelCase).__mro__[1:]  # make sure it's not the same module we've already patched
				    if issubclass(cls   , lowerCAmelCase) and cls.__module__ != DatasetBuilder.__module__
				]  # check it's not a standard builder from datasets.builder
				for module in parent_builder_modules:
								extend_module_for_streaming(lowerCAmelCase   , use_auth_token=builder.use_auth_token)
