

import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


lowerCamelCase   					=  get_logger(__name__)

# Datasets
lowerCamelCase   					=  '''https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'''
lowerCamelCase   					=  '''https://cdn-datasets.huggingface.co/datasets/datasets'''
lowerCamelCase   					=  '''https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'''

# Metrics
lowerCamelCase   					=  '''https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'''
lowerCamelCase   					=  '''https://cdn-datasets.huggingface.co/datasets/metric'''
lowerCamelCase   					=  '''https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'''

# Hub
lowerCamelCase   					=  os.environ.get('''HF_ENDPOINT''', '''https://huggingface.co''')
lowerCamelCase   					=  HF_ENDPOINT + '''/datasets/{repo_id}/resolve/{revision}/{path}'''
lowerCamelCase   					=  '''main'''

lowerCamelCase   					=  version.parse(platform.python_version())

if PY_VERSION < version.parse('''3.8'''):
		import importlib_metadata
else:
		import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
lowerCamelCase   					=  {'''1''', '''ON''', '''YES''', '''TRUE'''}
lowerCamelCase   					=  ENV_VARS_TRUE_VALUES.union({'''AUTO'''})


# Imports
lowerCamelCase   					=  version.parse(importlib_metadata.version('''dill'''))
lowerCamelCase   					=  version.parse(importlib_metadata.version('''pandas'''))
lowerCamelCase   					=  version.parse(importlib_metadata.version('''pyarrow'''))

lowerCamelCase   					=  os.environ.get('''USE_TF''', '''AUTO''').upper()
lowerCamelCase   					=  os.environ.get('''USE_TORCH''', '''AUTO''').upper()
lowerCamelCase   					=  os.environ.get('''USE_JAX''', '''AUTO''').upper()

lowerCamelCase   					=  '''N/A'''
lowerCamelCase   					=  False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
		lowerCamelCase   					=  importlib.util.find_spec('''torch''') is not None
		if TORCH_AVAILABLE:
				try:
						lowerCamelCase   					=  version.parse(importlib_metadata.version('''torch'''))
						logger.info(F'''PyTorch version {TORCH_VERSION} available.''')
				except importlib_metadata.PackageNotFoundError:
						pass
else:
		logger.info('''Disabling PyTorch because USE_TF is set''')

lowerCamelCase   					=  '''N/A'''
lowerCamelCase   					=  False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
		lowerCamelCase   					=  importlib.util.find_spec('''tensorflow''') is not None
		if TF_AVAILABLE:
				# For the metadata, we have to look for both tensorflow and tensorflow-cpu
				for package in [
				    "tensorflow",
				    "tensorflow-cpu",
				    "tensorflow-gpu",
				    "tf-nightly",
				    "tf-nightly-cpu",
				    "tf-nightly-gpu",
				    "intel-tensorflow",
				    "tensorflow-rocm",
				    "tensorflow-macos",
				]:
						try:
								lowerCamelCase   					=  version.parse(importlib_metadata.version(package))
						except importlib_metadata.PackageNotFoundError:
								continue
						else:
								break
				else:
						lowerCamelCase   					=  False
		if TF_AVAILABLE:
				if TF_VERSION.major < 2:
						logger.info(F'''TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.''')
						lowerCamelCase   					=  False
				else:
						logger.info(F'''TensorFlow version {TF_VERSION} available.''')
else:
		logger.info('''Disabling Tensorflow because USE_TORCH is set''')


lowerCamelCase   					=  '''N/A'''
lowerCamelCase   					=  False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
		lowerCamelCase   					=  importlib.util.find_spec('''jax''') is not None and importlib.util.find_spec('''jaxlib''') is not None
		if JAX_AVAILABLE:
				try:
						lowerCamelCase   					=  version.parse(importlib_metadata.version('''jax'''))
						logger.info(F'''JAX version {JAX_VERSION} available.''')
				except importlib_metadata.PackageNotFoundError:
						pass
else:
		logger.info('''Disabling JAX because USE_JAX is set to False''')


lowerCamelCase   					=  os.environ.get('''USE_BEAM''', '''AUTO''').upper()
lowerCamelCase   					=  '''N/A'''
lowerCamelCase   					=  False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
		try:
				lowerCamelCase   					=  version.parse(importlib_metadata.version('''apache_beam'''))
				lowerCamelCase   					=  True
				logger.info(F'''Apache Beam version {BEAM_VERSION} available.''')
		except importlib_metadata.PackageNotFoundError:
				pass
else:
		logger.info('''Disabling Apache Beam because USE_BEAM is set to False''')


# Optional tools for data loading
lowerCamelCase   					=  importlib.util.find_spec('''sqlalchemy''') is not None

# Optional tools for feature decoding
lowerCamelCase   					=  importlib.util.find_spec('''PIL''') is not None
lowerCamelCase   					=  importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.0.31''')
lowerCamelCase   					=  importlib.util.find_spec('''soundfile''') is not None and version.parse(
    importlib.import_module('''soundfile''').__libsndfile_version__
) >= version.parse('''1.1.0''')

# Optional compression tools
lowerCamelCase   					=  importlib.util.find_spec('''rarfile''') is not None
lowerCamelCase   					=  importlib.util.find_spec('''zstandard''') is not None
lowerCamelCase   					=  importlib.util.find_spec('''lz4''') is not None
lowerCamelCase   					=  importlib.util.find_spec('''py7zr''') is not None


# Cache location
lowerCamelCase   					=  '''~/.cache'''
lowerCamelCase   					=  os.getenv('''XDG_CACHE_HOME''', DEFAULT_XDG_CACHE_HOME)
lowerCamelCase   					=  os.path.join(XDG_CACHE_HOME, '''huggingface''')
lowerCamelCase   					=  os.path.expanduser(os.getenv('''HF_HOME''', DEFAULT_HF_CACHE_HOME))

lowerCamelCase   					=  os.path.join(HF_CACHE_HOME, '''datasets''')
lowerCamelCase   					=  Path(os.getenv('''HF_DATASETS_CACHE''', DEFAULT_HF_DATASETS_CACHE))

lowerCamelCase   					=  os.path.join(HF_CACHE_HOME, '''metrics''')
lowerCamelCase   					=  Path(os.getenv('''HF_METRICS_CACHE''', DEFAULT_HF_METRICS_CACHE))

lowerCamelCase   					=  os.path.join(HF_CACHE_HOME, '''modules''')
lowerCamelCase   					=  Path(os.getenv('''HF_MODULES_CACHE''', DEFAULT_HF_MODULES_CACHE))

lowerCamelCase   					=  '''downloads'''
lowerCamelCase   					=  os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
lowerCamelCase   					=  Path(os.getenv('''HF_DATASETS_DOWNLOADED_DATASETS_PATH''', DEFAULT_DOWNLOADED_DATASETS_PATH))

lowerCamelCase   					=  '''extracted'''
lowerCamelCase   					=  os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
lowerCamelCase   					=  Path(os.getenv('''HF_DATASETS_EXTRACTED_DATASETS_PATH''', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
lowerCamelCase   					=  (
    os.environ.get('''HF_UPDATE_DOWNLOAD_COUNTS''', '''AUTO''').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
lowerCamelCase   					=  1_0_0_0

# Size of the preloaded record batch in `Dataset.__iter__`
lowerCamelCase   					=  1_0

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
lowerCamelCase   					=  4 << 3_0

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
lowerCamelCase   					=  '''500MB'''

# Parquet configuration
lowerCamelCase   					=  1_0_0
lowerCamelCase   					=  1_0_0
lowerCamelCase   					=  1_0_0

# Offline mode
lowerCamelCase   					=  os.environ.get('''HF_DATASETS_OFFLINE''', '''AUTO''').upper() in ENV_VARS_TRUE_VALUES

# In-memory
lowerCamelCase   					=  0  # Disabled
lowerCamelCase   					=  float(os.environ.get('''HF_DATASETS_IN_MEMORY_MAX_SIZE''', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
lowerCamelCase   					=  '''dataset.arrow'''
lowerCamelCase   					=  '''indices.arrow'''
lowerCamelCase   					=  '''state.json'''
lowerCamelCase   					=  '''dataset_info.json'''
lowerCamelCase   					=  '''dataset_infos.json'''
lowerCamelCase   					=  '''LICENSE'''
lowerCamelCase   					=  '''metric_info.json'''
lowerCamelCase   					=  '''dataset_dict.json'''

lowerCamelCase   					=  '''datasets_modules'''

lowerCamelCase   					=  2_5_5

# Streaming
lowerCamelCase   					=  2_0
lowerCamelCase   					=  5

# Datasets without script
lowerCamelCase   					=  2_0_0
lowerCamelCase   					=  1_0
lowerCamelCase   					=  2_0_0

# Progress bars
lowerCamelCase   					=  0.05  # 20 progress updates per sec









import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase   					=  logging.get_logger(__name__)

lowerCamelCase   					=  {}





class      snake_case_      (					_a				):


							"""simple docstring"""





							__UpperCAmelCase      ="""fsmt_decoder"""




							def __init__(			self       ,    _A=0       ,    _A=0				):
												super().__init__()
												__lowerCAmelCase       =				vocab_size
												__lowerCAmelCase       =				bos_token_id





class      snake_case_      (					_a				):


							"""simple docstring"""





							__UpperCAmelCase      ="""fsmt"""
							__UpperCAmelCase      ={"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}




							def __init__(			self       ,    _A=["en", "de"]       ,    _A=4_2_0_2_4       ,    _A=4_2_0_2_4       ,    _A="relu"       ,    _A=1_0_2_4       ,    _A=2_0_0       ,    _A=1_0_2_4       ,    _A=4_0_9_6       ,    _A=1_2       ,    _A=1_6       ,    _A=0.0       ,    _A=4_0_9_6       ,    _A=1_2       ,    _A=1_6       ,    _A=0.0       ,    _A=0.0       ,    _A=0.1       ,    _A=0.0       ,    _A=0.02       ,    _A=2       ,    _A=True       ,    _A=True       ,    _A=False       ,    _A=5       ,    _A=1.0       ,    _A=False       ,    _A=True       ,    _A=1       ,    _A=0       ,    _A=2       ,    _A=2       ,    **_A       ,    ):
												__lowerCAmelCase       =				langs
												__lowerCAmelCase       =				src_vocab_size
												__lowerCAmelCase       =				tgt_vocab_size
												__lowerCAmelCase       =				d_model  # encoder_embed_dim and decoder_embed_dim

												__lowerCAmelCase       =				encoder_ffn_dim
												__lowerCAmelCase       =				__lowerCAmelCase       =				encoder_layers
												__lowerCAmelCase       =				encoder_attention_heads
												__lowerCAmelCase       =				encoder_layerdrop
												__lowerCAmelCase       =				decoder_layerdrop
												__lowerCAmelCase       =				decoder_ffn_dim
												__lowerCAmelCase       =				decoder_layers
												__lowerCAmelCase       =				decoder_attention_heads
												__lowerCAmelCase       =				max_position_embeddings
												__lowerCAmelCase       =				init_std  # Normal(0, this parameter)
												__lowerCAmelCase       =				activation_function

												__lowerCAmelCase       =				DecoderConfig(vocab_size=_A       ,    bos_token_id=_A				)
												if "decoder" in common_kwargs:
																	del common_kwargs["decoder"]

												__lowerCAmelCase       =				scale_embedding  # scale factor will be sqrt(d_model) if True

												# 3 Types of Dropout
												__lowerCAmelCase       =				attention_dropout
												__lowerCAmelCase       =				activation_dropout
												__lowerCAmelCase       =				dropout

												__lowerCAmelCase       =				use_cache
												super().__init__(
												    pad_token_id=_A       ,    bos_token_id=_A       ,    eos_token_id=_A       ,    decoder_start_token_id=_A       ,    is_encoder_decoder=_A       ,    tie_word_embeddings=_A       ,    forced_eos_token_id=_A       ,    max_length=_A       ,    num_beams=_A       ,    length_penalty=_A       ,    early_stopping=_A       ,    **_A       ,    )




							def 							A__	(			self				):
												__lowerCAmelCase       =				copy.deepcopy(self.__dict__				)
												__lowerCAmelCase       =				self.decoder.to_dict()
												__lowerCAmelCase       =				self.__class__.model_type
												return output






