
'''simple docstring'''





import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


UpperCAmelCase            = get_logger(__name__)

# Datasets
UpperCAmelCase            = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
UpperCAmelCase            = "https://cdn-datasets.huggingface.co/datasets/datasets"
UpperCAmelCase            = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"

# Metrics
UpperCAmelCase            = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"
UpperCAmelCase            = "https://cdn-datasets.huggingface.co/datasets/metric"
UpperCAmelCase            = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"

# Hub
UpperCAmelCase            = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
UpperCAmelCase            = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
UpperCAmelCase            = "main"

UpperCAmelCase            = version.parse(platform.python_version())

if PY_VERSION < version.parse("3.8"):
						import importlib_metadata
else:
						import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
UpperCAmelCase            = {"1", "ON", "YES", "TRUE"}
UpperCAmelCase            = ENV_VARS_TRUE_VALUES.union({"AUTO"})


# Imports
UpperCAmelCase            = version.parse(importlib_metadata.version("dill"))
UpperCAmelCase            = version.parse(importlib_metadata.version("pandas"))
UpperCAmelCase            = version.parse(importlib_metadata.version("pyarrow"))

UpperCAmelCase            = os.environ.get("USE_TF", "AUTO").upper()
UpperCAmelCase            = os.environ.get("USE_TORCH", "AUTO").upper()
UpperCAmelCase            = os.environ.get("USE_JAX", "AUTO").upper()

UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("torch") is not None
						if TORCH_AVAILABLE:
												try:
																		UpperCAmelCase            = version.parse(importlib_metadata.version("torch"))
																		logger.info(f'''PyTorch version {TORCH_VERSION} available.''')
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling PyTorch because USE_TF is set")

UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("tensorflow") is not None
						if TF_AVAILABLE:
												# For the metadata, we have to look for both tensorflow and tensorflow-cpu
												for package in [
												    "tensorflow",
												    "tensorflow-cpu",
												    "tensorflow-gpu",
												    "tf-nightly",
												    "tf-nightly-cpu",
												    "tf-nightly-gpu",
												    "intel-tensorflow",
												    "tensorflow-rocm",
												    "tensorflow-macos",
												]:
																		try:
																								UpperCAmelCase            = version.parse(importlib_metadata.version(package))
																		except importlib_metadata.PackageNotFoundError:
																								continue
																		else:
																								break
												else:
																		UpperCAmelCase            = False
						if TF_AVAILABLE:
												if TF_VERSION.major < 2:
																		logger.info(f'''TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.''')
																		UpperCAmelCase            = False
												else:
																		logger.info(f'''TensorFlow version {TF_VERSION} available.''')
else:
						logger.info("Disabling Tensorflow because USE_TORCH is set")


UpperCAmelCase            = "N/A"
UpperCAmelCase            = False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
						UpperCAmelCase            = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
						if JAX_AVAILABLE:
												try:
																		UpperCAmelCase            = version.parse(importlib_metadata.version("jax"))
																		logger.info(f'''JAX version {JAX_VERSION} available.''')
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling JAX because USE_JAX is set to False")


UpperCAmelCase            = os.environ.get("USE_BEAM", "AUTO").upper()
UpperCAmelCase            = "N/A"
UpperCAmelCase            = False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
						try:
												UpperCAmelCase            = version.parse(importlib_metadata.version("apache_beam"))
												UpperCAmelCase            = True
												logger.info(f'''Apache Beam version {BEAM_VERSION} available.''')
						except importlib_metadata.PackageNotFoundError:
												pass
else:
						logger.info("Disabling Apache Beam because USE_BEAM is set to False")


# Optional tools for data loading
UpperCAmelCase            = importlib.util.find_spec("sqlalchemy") is not None

# Optional tools for feature decoding
UpperCAmelCase            = importlib.util.find_spec("PIL") is not None
UpperCAmelCase            = importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.0.31")
UpperCAmelCase            = importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.1.0")

# Optional compression tools
UpperCAmelCase            = importlib.util.find_spec("rarfile") is not None
UpperCAmelCase            = importlib.util.find_spec("zstandard") is not None
UpperCAmelCase            = importlib.util.find_spec("lz4") is not None
UpperCAmelCase            = importlib.util.find_spec("py7zr") is not None


# Cache location
UpperCAmelCase            = "~/.cache"
UpperCAmelCase            = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
UpperCAmelCase            = os.path.join(XDG_CACHE_HOME, "huggingface")
UpperCAmelCase            = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "datasets")
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "metrics")
UpperCAmelCase            = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE))

UpperCAmelCase            = os.path.join(HF_CACHE_HOME, "modules")
UpperCAmelCase            = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))

UpperCAmelCase            = "downloads"
UpperCAmelCase            = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))

UpperCAmelCase            = "extracted"
UpperCAmelCase            = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
UpperCAmelCase            = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
UpperCAmelCase            = (
    os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
UpperCAmelCase            = 1_000

# Size of the preloaded record batch in `Dataset.__iter__`
UpperCAmelCase            = 10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
UpperCAmelCase            = 4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
UpperCAmelCase            = "500MB"

# Parquet configuration
UpperCAmelCase            = 100
UpperCAmelCase            = 100
UpperCAmelCase            = 100

# Offline mode
UpperCAmelCase            = os.environ.get("HF_DATASETS_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES

# In-memory
UpperCAmelCase            = 0  # Disabled
UpperCAmelCase            = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
UpperCAmelCase            = "dataset.arrow"
UpperCAmelCase            = "indices.arrow"
UpperCAmelCase            = "state.json"
UpperCAmelCase            = "dataset_info.json"
UpperCAmelCase            = "dataset_infos.json"
UpperCAmelCase            = "LICENSE"
UpperCAmelCase            = "metric_info.json"
UpperCAmelCase            = "dataset_dict.json"

UpperCAmelCase            = "datasets_modules"

UpperCAmelCase            = 255

# Streaming
UpperCAmelCase            = 20
UpperCAmelCase            = 5

# Datasets without script
UpperCAmelCase            = 200
UpperCAmelCase            = 10
UpperCAmelCase            = 200

# Progress bars
UpperCAmelCase            = 0.05  # 20 progress updates per sec



"""simple docstring"""




import argparse
import collections
import importlib.util
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
A__   :						List[str] 							=     """src/diffusers"""
A__   :						List[str] 							=     """docs/source/en"""
A__   :						List[Any] 							=     """."""



def 				a__			(					lowerCAmelCase						:							List[str]							,     lowerCAmelCase						:							Any							,     lowerCAmelCase						:							List[Any]		):






     '''simple docstring'''
     with open(lowerCAmelCase							,     "r"							,     encoding="utf-8"							,     newline="\n"		) as f:
          UpperCAmelCase__				:			List[str]							=    f.readlines()
     # Find the start prompt.
     UpperCAmelCase__				:			Union[str, Any]							=    0
     while not lines[start_index].startswith(lowerCAmelCase		):
          start_index += 1
     start_index += 1

     UpperCAmelCase__				:			List[Any]							=    start_index
     while not lines[end_index].startswith(lowerCAmelCase		):
          end_index += 1
     end_index -= 1

     while len(lines[start_index]		) <= 1:
          start_index += 1
     while len(lines[end_index]		) <= 1:
          end_index -= 1
     end_index += 1
     return "".join(lines[start_index:end_index]		), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
A__   :						Tuple 							=     """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
A__   :						Any 							=     re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
A__   :						Tuple 							=     re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
A__   :						List[Any] 							=     re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the diffusers module imported is the one in the repo.
A__   :						List[Any] 							=     importlib.util.spec_from_file_location(
    """diffusers""",
    os.path.join(TRANSFORMERS_PATH, """__init__.py"""),
    submodule_search_locations=[TRANSFORMERS_PATH],
)
A__   :						Dict 							=     spec.loader.load_module()



def 				a__			(					lowerCAmelCase						:							List[str]		):






     '''simple docstring'''
     UpperCAmelCase__				:			Dict							=    re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"							,     lowerCAmelCase		)
     return [m.group(0		) for m in matches]



def 				a__			(					lowerCAmelCase						:							Tuple							,     lowerCAmelCase						:							Union[str, Any]		):






     '''simple docstring'''
     UpperCAmelCase__				:			List[Any]							=    2 if text == "✅" or text == "❌" else len(lowerCAmelCase		)
     UpperCAmelCase__				:			Any							=    (width - text_length) // 2
     UpperCAmelCase__				:			List[Any]							=    width - text_length - left_indent
     return " " * left_indent + text + " " * right_indent



def 				a__			(					):






     '''simple docstring'''
     UpperCAmelCase__				:			int							=    diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
     UpperCAmelCase__				:			Optional[int]							=    {
         name: config_mapping_names[code]
         for code, name in diffusers_module.MODEL_NAMES_MAPPING.items()
         if code in config_mapping_names
     }
     UpperCAmelCase__				:			Optional[int]							=    {name: config.replace("ConfigMixin"							,     ""		) for name, config in model_name_to_config.items()}

     # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
     UpperCAmelCase__				:			str							=    collections.defaultdict(lowerCAmelCase		)
     UpperCAmelCase__				:			Optional[int]							=    collections.defaultdict(lowerCAmelCase		)
     UpperCAmelCase__				:			str							=    collections.defaultdict(lowerCAmelCase		)
     UpperCAmelCase__				:			str							=    collections.defaultdict(lowerCAmelCase		)
     UpperCAmelCase__				:			Union[str, Any]							=    collections.defaultdict(lowerCAmelCase		)

     # Let's lookup through all diffusers object (once).
     for attr_name in dir(lowerCAmelCase		):
          UpperCAmelCase__				:			List[str]							=    None
          if attr_name.endswith("Tokenizer"		):
               UpperCAmelCase__				:			Union[str, Any]							=    slow_tokenizers
               UpperCAmelCase__				:			Optional[Any]							=    attr_name[:-9]
          elif attr_name.endswith("TokenizerFast"		):
               UpperCAmelCase__				:			List[str]							=    fast_tokenizers
               UpperCAmelCase__				:			Union[str, Any]							=    attr_name[:-13]
          elif _re_tf_models.match(lowerCAmelCase		) is not None:
               UpperCAmelCase__				:			Union[str, Any]							=    tf_models
               UpperCAmelCase__				:			Optional[int]							=    _re_tf_models.match(lowerCAmelCase		).groups()[0]
          elif _re_flax_models.match(lowerCAmelCase		) is not None:
               UpperCAmelCase__				:			List[Any]							=    flax_models
               UpperCAmelCase__				:			Any							=    _re_flax_models.match(lowerCAmelCase		).groups()[0]
          elif _re_pt_models.match(lowerCAmelCase		) is not None:
               UpperCAmelCase__				:			str							=    pt_models
               UpperCAmelCase__				:			List[str]							=    _re_pt_models.match(lowerCAmelCase		).groups()[0]

          if lookup_dict is not None:
               while len(lowerCAmelCase		) > 0:
                    if attr_name in model_name_to_prefix.values():
                         UpperCAmelCase__				:			Optional[Any]							=    True
                         break
                    # Try again after removing the last word in the name
                    UpperCAmelCase__				:			Union[str, Any]							=    "".join(camel_case_split(lowerCAmelCase		)[:-1]		)

    # Let's build that table!
     UpperCAmelCase__				:			Tuple							=    list(model_name_to_config.keys()		)
     model_names.sort(key=str.lower		)
     UpperCAmelCase__				:			Dict							=    ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
     # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
     UpperCAmelCase__				:			List[str]							=    [len(lowerCAmelCase		) + 2 for c in columns]
     UpperCAmelCase__				:			Union[str, Any]							=    max([len(lowerCAmelCase		) for name in model_names]		) + 2

     # Build the table per se
     UpperCAmelCase__				:			str							=    "|" + "|".join([_center_text(lowerCAmelCase							,     lowerCAmelCase		) for c, w in zip(lowerCAmelCase							,     lowerCAmelCase		)]		) + "|\n"
     # Use ":-----:" format to center-aligned table cell texts
     table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]		) + "|\n"

     UpperCAmelCase__				:			List[Any]							=    {True: "✅", False: "❌"}
     for name in model_names:
          UpperCAmelCase__				:			Optional[Any]							=    model_name_to_prefix[name]
          UpperCAmelCase__				:			List[Any]							=    [
              name,
              check[slow_tokenizers[prefix]],
              check[fast_tokenizers[prefix]],
              check[pt_models[prefix]],
              check[tf_models[prefix]],
              check[flax_models[prefix]],
          ]
          table += "|" + "|".join([_center_text(lowerCAmelCase							,     lowerCAmelCase		) for l, w in zip(lowerCAmelCase							,     lowerCAmelCase		)]		) + "|\n"
     return table



def 				a__			(					lowerCAmelCase						:							Union[str, Any]=False		):






     '''simple docstring'''
     UpperCAmelCase__				,						UpperCAmelCase__				,						UpperCAmelCase__				,						UpperCAmelCase__				:			int							=    _find_text_in_file(
         filename=os.path.join(lowerCAmelCase							,     "index.mdx"		)							,     start_prompt="<!--This table is updated automatically from the auto modules"							,     end_prompt="<!-- End table-->"							,     )
     UpperCAmelCase__				:			Optional[Any]							=    get_model_table_from_auto_modules()

     if current_table != new_table:
          if overwrite:
               with open(os.path.join(lowerCAmelCase							,     "index.mdx"		)							,     "w"							,     encoding="utf-8"							,     newline="\n"		) as f:
                    f.writelines(lines[:start_index] + [new_table] + lines[end_index:]		)
          else:
               raise ValueError(
                   "The model table in the `index.mdx` has not been updated. Run `make fix-copies` to fix this."		)


if __name__ == "__main__":
    A__   :						Union[str, Any] 							=     argparse.ArgumentParser()
    parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
    A__   :						str 							=     parser.parse_args()

    check_model_table(args.fix_and_overwrite)



