'''simple docstring'''

import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
						import importlib.resources as pkg_resources
except ImportError:
						# Try backported to PY<37 `importlib_resources`.
						import importlib_resources as pkg_resources

from . import resources


A_	   = "https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
A_	   = f"{BASE_REF_URL}/{__file__}"
A_	   = logging.getLogger(__name__)


def        UpperCAmelCase__   (				_lowerCAmelCase    :			str)    ->		Tuple[Any, str]:



						"""simple docstring"""

						snake_case :       str				=    pkg_resources.read_text(_lowerCAmelCase,      _lowerCAmelCase)
						return yaml.safe_load(_lowerCAmelCase), f"""{BASE_REF_URL}/resources/{resource}"""


A_   , A_	   = load_yaml_resource("readme_structure.yaml")

A_	   = [
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
A_	   = Tuple[dict, List[str], List[str]]




class 		SCREAMING_SNAKE_CASE_					:



			"""simple docstring"""






			def __init__(							self				,			snake_case_				,			snake_case_				,			snake_case_ = None				,			snake_case_ = False					):
									snake_case :       Any				=    name
									snake_case :       Any				=    level
									snake_case :       Union[str, Any]				=    lines
									snake_case :       str				=    ""
									snake_case :       Optional[int]				=    True
									snake_case :       str				=    {}
									snake_case :       int				=    []
									snake_case :       Tuple				=    []
									if self.lines is not None:
															self.parse(suppress_parsing_errors=snake_case_					)



			def   lowerCamelCase (							self				,			snake_case_ = False					):
									snake_case :       str				=    ""
									snake_case :       Dict				=    []
									snake_case :       List[Any]				=    False
									for line in self.lines:
															if line.strip(" \n"					) == "":
																					continue
															elif line.strip(" \n"					)[:3] == "```":
																					snake_case :       Dict				=    not code_start
															elif line.split()[0] == self.level + "#" and not code_start:
																					if current_sub_level != "":
																											snake_case :       Optional[int]				=    Section(snake_case_				,			self.level + "#"				,			snake_case_					)
																											snake_case :       Dict				=    []
																					else:
																											if current_lines != []:
																																	self.text += "".join(snake_case_					).strip()
																																	if self.text != "" and self.text not in FILLER_TEXT:
																																							snake_case :       int				=    False
																																	snake_case :       List[Any]				=    []

																					snake_case :       Union[str, Any]				=    " ".join(line.split()[1:]					).strip(" \n"					)
															else:
																					current_lines.append(snake_case_					)
									else:
															if current_sub_level != "":
																					if current_sub_level in self.content:
																											self.parsing_error_list.append(
																											    F"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""					)
																					snake_case :       Union[str, Any]				=    Section(snake_case_				,			self.level + "#"				,			snake_case_					)
															else:
																					if current_lines != []:
																											self.text += "".join(snake_case_					).strip()
																											if self.text != "" and self.text not in FILLER_TEXT:
																																	snake_case :       Optional[int]				=    False

									if self.level == "" and not suppress_parsing_errors:
															if self.parsing_error_list != [] or self.parsing_warning_list != []:
																					snake_case :       List[str]				=    "\n".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list					)
																					snake_case :       Optional[int]				=    F"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
																					raise ValueError(snake_case_					)



			def   lowerCamelCase (							self				,			snake_case_					):
									snake_case :       List[str]				=    []
									snake_case :       int				=    []
									if structure["allow_empty"] is False:
															# If content is expected
															if self.is_empty_text and self.content == {}:
																					# If no content is found, mention it in the error_list
																					error_list.append(F"""Expected some content in section `{self.name}` but it is empty."""					)

									if structure["allow_empty_text"] is False:
															# If some text is expected
															if self.is_empty_text:
																					# If no text is found, mention it in the error_list
																					error_list.append(
																					    F"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""					)
        # Subsections Validation
									if structure["subsections"] is not None:
															# If subsections are expected
															if self.content == {}:
																					# If no subsections are present
																					snake_case :       str				=    [subsection["name"] for subsection in structure["subsections"]]
																					# Mention the expected values in the error_list
																					error_list.append(
																					    F"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]					)}. Found 'None'."""					)
															else:
																					# If some subsections are present
																					snake_case :       int				=    [subsection["name"] for subsection in structure["subsections"]]
																					snake_case :       str				=    False
																					for idx, name in enumerate(snake_case_					):
																											if name not in self.content:
																																	# If the expected subsection is not present
																																	error_list.append(F"""Section `{self.name}` is missing subsection: `{name}`."""					)
																																	snake_case :       str				=    True
																											else:
																																	# If the subsection is present, validate subsection, return the result
																																	# and concat the errors from subsection to section error_list

																																	# Skip sublevel validation if current level is `###`
																																	if self.level == "###":
																																							continue
																																	else:
																																							snake_case				,					snake_case				,					snake_case :       Optional[Any]				=    self.content[name].validate(
																																							    structure["subsections"][idx]					)
																																	error_list += subsec_error_list
																																	warning_list += subsec_warning_list

																					if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																											for name in self.content:
																																	if name not in structure_names:
																																							# If an extra subsection is present
																																							warning_list.append(
																																							    F"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""					)
									if error_list:
															# If there are errors, do not return the dictionary as it is invalid
															return {}, error_list, warning_list
									else:
															return self.to_dict(), error_list, warning_list






			def   lowerCamelCase (							self					):
									return {
									    "name": self.name,
									    "text": self.text,
									    "is_empty_text": self.is_empty_text,
									    "subsections": [value.to_dict() for value in self.content.values()],
									}







class 		SCREAMING_SNAKE_CASE_					(_UpperCAmelCase      ):  # Level 0



			"""simple docstring"""






			def __init__(							self				,			snake_case_				,			snake_case_				,			snake_case_ = None				,			snake_case_ = False					):
									super().__init__(name=snake_case_				,			level=""					)  # Not using lines here as we need to use a child class parse
									snake_case :       int				=    structure
									snake_case :       Dict				=    -2
									snake_case :       int				=    0
									snake_case :       Optional[Any]				=    lines
									if self.lines is not None:
															self.parse(suppress_parsing_errors=snake_case_					)



			def   lowerCamelCase (							self					):
									if self.structure is None:
															snake_case				,					snake_case				,					snake_case :       Tuple				=    self._validate(snake_case_					)
									else:
															snake_case				,					snake_case				,					snake_case :       Union[str, Any]				=    self._validate(self.structure					)
									if error_list != [] or warning_list != []:
															snake_case :       int				=    "\n".join(["-\t" + x for x in error_list + warning_list]					)
															snake_case :       Union[str, Any]				=    F"""The following issues were found for the README at `{self.name}`:\n""" + errors
															raise ValueError(snake_case_					)



			@classmethod
			def   lowerCamelCase (							cls				,			snake_case_				,			snake_case_ = None				,			snake_case_ = False					):
									with open(snake_case_				,			encoding="utf-8"					) as f:
															snake_case :       List[str]				=    f.readlines()
									return cls(snake_case_				,			snake_case_				,			snake_case_				,			suppress_parsing_errors=snake_case_					)



			@classmethod
			def   lowerCamelCase (							cls				,			snake_case_				,			snake_case_ = None				,			snake_case_ = "root"				,			snake_case_ = False					):
									snake_case :       Optional[Any]				=    string.split("\n"					)
									return cls(snake_case_				,			snake_case_				,			snake_case_				,			suppress_parsing_errors=snake_case_					)



			def   lowerCamelCase (							self				,			snake_case_ = False					):
									# Skip Tags
									snake_case :       str				=    0

									for line in self.lines:
															self.yaml_tags_line_count += 1
															if line.strip(" \n"					) == "---":
																					self.tag_count += 1
																					if self.tag_count == 2:
																											break
															line_count += 1
									if self.tag_count == 2:
															snake_case :       str				=    self.lines[line_count + 1 :]  # Get the last + 1 th item.
									else:
															snake_case :       Optional[Any]				=    self.lines[self.tag_count :]
									super().parse(suppress_parsing_errors=snake_case_					)



			def __str__(							self					):
									return str(self.to_dict()					)






			def   lowerCamelCase (							self				,			snake_case_					):
									snake_case :       str				=    []
									snake_case :       List[str]				=    []
									if self.yaml_tags_line_count == 0:
															warning_list.append("Empty YAML markers are present in the README."					)
									elif self.tag_count == 0:
															warning_list.append("No YAML markers are present in the README."					)
									elif self.tag_count == 1:
															warning_list.append("Only the start of YAML tags present in the README."					)
									# Check how many first level sections are present.
									snake_case :       Optional[int]				=    len(self.content.keys()					)
									if num_first_level_keys > 1:
															# If more than one, add to the error list, continue
															error_list.append(
															    F"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()					)]					)}. Only one heading is expected. Skipping further validation for this README."""					)
									elif num_first_level_keys < 1:
															# If less than one, append error.
															error_list.append(
															    "The README has no first-level headings. One heading is expected. Skipping further validation for this README."					)

									else:
															# If one exactly
															snake_case :       Any				=    list(self.content.keys()					)[0]  # Get the key
															if start_key.startswith("Dataset Card for"					):  # Check correct start
																					# If the starting is correct, validate all the sections
																					snake_case				,					snake_case				,					snake_case :       int				=    self.content[start_key].validate(
																					    readme_structure["subsections"][0]					)
																					error_list += sec_error_list
																					warning_list += sec_warning_list
															else:
																					# If not found, append error
																					error_list.append(
																					    "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."					)
									if error_list:
															# If there are errors, do not return the dictionary as it is invalid
															return {}, error_list, warning_list
									else:
															return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
						from argparse import ArgumentParser

						A_	   = ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
						ap.add_argument("readme_filepath")
						A_	   = ap.parse_args()
						A_	   = Path(args.readme_filepath)
						A_	   = ReadMe.from_readme(readme_filepath)






'''simple docstring'''

import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 		SCREAMING_SNAKE_CASE_					(_UpperCAmelCase      ):



			"""simple docstring"""






			__magic_name__ :				Dict   							=       ["""image_processor""", """tokenizer"""]
			__magic_name__ :				Union[str, Any]   							=       """VideoMAEImageProcessor"""
			__magic_name__ :				List[str]   							=       ("""CLIPTokenizer""", """CLIPTokenizerFast""")



			def __init__(							self				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):
									snake_case :       int				=    None
									if "feature_extractor" in kwargs:
															warnings.warn(
															    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
															    " instead."				,			snake_case_				,			)
															snake_case :       Optional[int]				=    kwargs.pop("feature_extractor"					)

									snake_case :       Union[str, Any]				=    image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
															raise ValueError("You need to specify an `image_processor`."					)
									if tokenizer is None:
															raise ValueError("You need to specify a `tokenizer`."					)

									super().__init__(snake_case_				,			snake_case_					)
									snake_case :       List[Any]				=    self.image_processor



			def __call__(							self				,			snake_case_=None				,			snake_case_=None				,			snake_case_=None				,			**snake_case_					):

									if text is None and videos is None:
															raise ValueError("You have to specify either text or videos. Both cannot be none."					)

									if text is not None:
															snake_case :       List[Any]				=    self.tokenizer(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if videos is not None:
															snake_case :       List[Any]				=    self.image_processor(snake_case_				,			return_tensors=snake_case_				,			**snake_case_					)

									if text is not None and videos is not None:
															snake_case :       Tuple				=    image_features.pixel_values
															return encoding
									elif text is not None:
															return encoding
									else:
															return BatchEncoding(data=dict(**snake_case_					)				,			tensor_type=snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.batch_decode(*snake_case_				,			**snake_case_					)



			def   lowerCamelCase (							self				,			*snake_case_				,			**snake_case_					):
									return self.tokenizer.decode(*snake_case_				,			**snake_case_					)



			@property
			def   lowerCamelCase (							self					):
									return ["input_ids", "attention_mask", "position_ids", "pixel_values"]



			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."				,			snake_case_				,			)
									return self.image_processor_class






			@property
			def   lowerCamelCase (							self					):
									warnings.warn(
									    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."				,			snake_case_				,			)
									return self.image_processor





