
import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
				import importlib.resources as pkg_resources
except ImportError:
				# Try backported to PY<37 `importlib_resources`.
				import importlib_resources as pkg_resources

from . import resources


lowercase__				:				Any						=      "https://github.com/huggingface/datasets/tree/main/src/datasets/utils"
lowercase__				:				str						=      F"{BASE_REF_URL}/{__file__}"
lowercase__				:				str						=      logging.getLogger(__name__)




def 						A_			(				snake_case				: str			)   ->     Tuple[Any, str]:



						'''simple docstring'''

						__UpperCamelCase  	=      pkg_resources.read_text(snake_case			,   snake_case			)
						return yaml.safe_load(snake_case			), f"{BASE_REF_URL}/resources/{resource}"


lowercase__		,	lowercase__				:				Any						=      load_yaml_resource("readme_structure.yaml")

lowercase__				:				List[Any]						=      [
    "[Needs More Information]",
    "[More Information Needed]",
    "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)",
]

# Dictionary representation of section/readme, error_list, warning_list
lowercase__				:				Any						=      Tuple[dict, List[str], List[str]]






class 							SCREAMING_SNAKE_CASE__     :

							"""simple docstring"""






							def __init__(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_ = None  ,						SCREAMING_SNAKE_CASE_ = False      )-> int:





													'''simple docstring'''






													__UpperCamelCase  	=      name
													__UpperCamelCase  	=      level
													__UpperCamelCase  	=      lines
													__UpperCamelCase  	=      ''''''
													__UpperCamelCase  	=      True
													__UpperCamelCase  	=      {}
													__UpperCamelCase  	=      []
													__UpperCamelCase  	=      []
													if self.lines is not None:
																			self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_ = False      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      ''''''
													__UpperCamelCase  	=      []
													__UpperCamelCase  	=      False
													for line in self.lines:
																			if line.strip(''' \n'''      ) == "":
																									continue
																			elif line.strip(''' \n'''      )[:3] == "```":
																									__UpperCamelCase  	=      not code_start
																			elif line.split()[0] == self.level + "#" and not code_start:
																									if current_sub_level != "":
																															__UpperCamelCase  	=      Section(SCREAMING_SNAKE_CASE_  ,						self.level + '''#'''  ,						SCREAMING_SNAKE_CASE_      )
																															__UpperCamelCase  	=      []
																									else:
																															if current_lines != []:
																																					self.text += "".join(SCREAMING_SNAKE_CASE_      ).strip()
																																					if self.text != "" and self.text not in FILLER_TEXT:
																																											__UpperCamelCase  	=      False
																																					__UpperCamelCase  	=      []

																									__UpperCamelCase  	=      ''' '''.join(line.split()[1:]      ).strip(''' \n'''      )
																			else:
																									current_lines.append(SCREAMING_SNAKE_CASE_      )
													else:
																			if current_sub_level != "":
																									if current_sub_level in self.content:
																															self.parsing_error_list.append(
																															    F"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."      )
																									__UpperCamelCase  	=      Section(SCREAMING_SNAKE_CASE_  ,						self.level + '''#'''  ,						SCREAMING_SNAKE_CASE_      )
																			else:
																									if current_lines != []:
																															self.text += "".join(SCREAMING_SNAKE_CASE_      ).strip()
																															if self.text != "" and self.text not in FILLER_TEXT:
																																					__UpperCamelCase  	=      False

													if self.level == "" and not suppress_parsing_errors:
																			if self.parsing_error_list != [] or self.parsing_warning_list != []:
																									__UpperCamelCase  	=      __UpperCamelCase  	=      '''\n'''.join('''-\t''' + x for x in self.parsing_error_list + self.parsing_warning_list      )
																									__UpperCamelCase  	=      F"The following issues were found while parsing the README at `{self.name}`:\n" + errors
																									raise ValueError(SCREAMING_SNAKE_CASE_      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> ReadmeValidatorOutput:





													'''simple docstring'''






													__UpperCamelCase  	=      []
													__UpperCamelCase  	=      []
													if structure["allow_empty"] is False:
																			# If content is expected
																			if self.is_empty_text and self.content == {}:
																									# If no content is found, mention it in the error_list
																									error_list.append(F"Expected some content in section `{self.name}` but it is empty."      )

													if structure["allow_empty_text"] is False:
																			# If some text is expected
																			if self.is_empty_text:
																									# If no text is found, mention it in the error_list
																									error_list.append(
																									    F"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."      )
        # Subsections Validation
													if structure["subsections"] is not None:
																			# If subsections are expected
																			if self.content == {}:
																									# If no subsections are present
																									__UpperCamelCase  	=      [subsection['''name'''] for subsection in structure['''subsections''']]
																									# Mention the expected values in the error_list
																									error_list.append(
																									    F"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]      )}. Found 'None'."      )
																			else:
																									# If some subsections are present
																									__UpperCamelCase  	=      [subsection['''name'''] for subsection in structure['''subsections''']]
																									__UpperCamelCase  	=      False
																									for idx, name in enumerate(SCREAMING_SNAKE_CASE_      ):
																															if name not in self.content:
																																					# If the expected subsection is not present
																																					error_list.append(F"Section `{self.name}` is missing subsection: `{name}`."      )
																																					__UpperCamelCase  	=      True
																															else:
																																					# If the subsection is present, validate subsection, return the result
																																					# and concat the errors from subsection to section error_list

																																					# Skip sublevel validation if current level is `###`
																																					if self.level == "###":
																																											continue
																																					else:
																																											__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase  	=      self.content[name].validate(
																																											    structure['''subsections'''][idx]      )
																																					error_list += subsec_error_list
																																					warning_list += subsec_warning_list

																									if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																															for name in self.content:
																																					if name not in structure_names:
																																											# If an extra subsection is present
																																											warning_list.append(
																																											    F"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."      )
													if error_list:
																			# If there are errors, do not return the dictionary as it is invalid
																			return {}, error_list, warning_list
													else:
																			return self.to_dict(), error_list, warning_list



							def      A__		(	self      )-> dict:





													'''simple docstring'''






													return {
													    "name": self.name,
													    "text": self.text,
													    "is_empty_text": self.is_empty_text,
													    "subsections": [value.to_dict() for value in self.content.values()],
													}






class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_     ):  # Level 0

							"""simple docstring"""






							def __init__(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_ = None  ,						SCREAMING_SNAKE_CASE_ = False      )-> List[str]:





													'''simple docstring'''






													super().__init__(name=SCREAMING_SNAKE_CASE_  ,						level=''''''      )  # Not using lines here as we need to use a child class parse
													__UpperCamelCase  	=      structure
													__UpperCamelCase  	=      -2
													__UpperCamelCase  	=      0
													__UpperCamelCase  	=      lines
													if self.lines is not None:
																			self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													if self.structure is None:
																			__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase  	=      self._validate(SCREAMING_SNAKE_CASE_      )
													else:
																			__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase  	=      self._validate(self.structure      )
													if error_list != [] or warning_list != []:
																			__UpperCamelCase  	=      '''\n'''.join(['''-\t''' + x for x in error_list + warning_list]      )
																			__UpperCamelCase  	=      F"The following issues were found for the README at `{self.name}`:\n" + errors
																			raise ValueError(SCREAMING_SNAKE_CASE_      )



							@classmethod
							def      A__		(	cls  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_ = None  ,						SCREAMING_SNAKE_CASE_ = False      )-> Union[str, Any]:





													'''simple docstring'''






													with open(SCREAMING_SNAKE_CASE_  ,						encoding='''utf-8'''      ) as f:
																			__UpperCamelCase  	=      f.readlines()
													return cls(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )



							@classmethod
							def      A__		(	cls  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_ = None  ,						SCREAMING_SNAKE_CASE_ = "root"  ,						SCREAMING_SNAKE_CASE_ = False      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      string.split('''\n'''      )
													return cls(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_ = False      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      0

													for line in self.lines:
																			self.yaml_tags_line_count += 1
																			if line.strip(''' \n'''      ) == "---":
																									self.tag_count += 1
																									if self.tag_count == 2:
																															break
																			line_count += 1
													if self.tag_count == 2:
																			__UpperCamelCase  	=      self.lines[line_count + 1 :]  # Get the last + 1 th item.
													else:
																			__UpperCamelCase  	=      self.lines[self.tag_count :]
													super().parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )



							def __str__(	self      )-> Optional[Any]:





													'''simple docstring'''






													return str(self.to_dict()      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> Any:





													'''simple docstring'''






													__UpperCamelCase  	=      []
													__UpperCamelCase  	=      []
													if self.yaml_tags_line_count == 0:
																			warning_list.append('''Empty YAML markers are present in the README.'''      )
													elif self.tag_count == 0:
																			warning_list.append('''No YAML markers are present in the README.'''      )
													elif self.tag_count == 1:
																			warning_list.append('''Only the start of YAML tags present in the README.'''      )
													# Check how many first level sections are present.
													__UpperCamelCase  	=      len(self.content.keys()      )
													if num_first_level_keys > 1:
																			# If more than one, add to the error list, continue
																			error_list.append(
																			    F"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()      )]      )}. Only one heading is expected. Skipping further validation for this README."      )
													elif num_first_level_keys < 1:
																			# If less than one, append error.
																			error_list.append(
																			    '''The README has no first-level headings. One heading is expected. Skipping further validation for this README.'''      )

													else:
																			# If one exactly
																			__UpperCamelCase  	=      list(self.content.keys()      )[0]  # Get the key
																			if start_key.startswith('''Dataset Card for'''      ):  # Check correct start
																									# If the starting is correct, validate all the sections
																									__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase  	=      self.content[start_key].validate(
																									    readme_structure['''subsections'''][0]      )
																									error_list += sec_error_list
																									warning_list += sec_warning_list
																			else:
																									# If not found, append error
																									error_list.append(
																									    '''No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'''      )
													if error_list:
																			# If there are errors, do not return the dictionary as it is invalid
																			return {}, error_list, warning_list
													else:
																			return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
				from argparse import ArgumentParser

				lowercase__				:				Dict						=      ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.")
				ap.add_argument("readme_filepath")
				lowercase__				:				Optional[Any]						=      ap.parse_args()
				lowercase__				:				Any						=      Path(args.readme_filepath)
				lowercase__				:				str						=      ReadMe.from_readme(readme_filepath)








'''simple docstring'''
import subprocess
import sys
import warnings
from argparse import ArgumentParser
from pathlib import Path

from packaging import version

from .. import AutoFeatureExtractor, AutoImageProcessor, AutoProcessor, AutoTokenizer
from ..utils import logging
from ..utils.import_utils import is_optimum_available
from .convert import export, validate_model_outputs
from .features import FeaturesManager
from .utils import get_preprocessor


lowerCAmelCase__  =     '1.5.0'

lowerCAmelCase__  =     ['vision-encoder-decoder']



def 				lowerCAmelCase__	(       UpperCAmelCase     ):






       """simple docstring"""





       if is_optimum_available():
              from optimum.version import __version__ as optimum_version

              snake_case__					:	Dict          =					version.parse(UpperCAmelCase     )
              if parsed_optimum_version < version.parse(UpperCAmelCase     ):
                     raise RuntimeError(
                         f"""transformers.onnx requires optimum >= {MIN_OPTIMUM_VERSION} but {optimum_version} is installed. You """
                         """can upgrade optimum by running: pip install -U optimum[exporters]"""     )
       else:
              raise RuntimeError(
                  """transformers.onnx requires optimum to run, you can install the library by running: pip install """
                  """optimum[exporters]"""     )
       snake_case__					:	Union[str, Any]          =					[
           sys.executable,
           """-m""",
           """optimum.exporters.onnx""",
           f"""--model {args.model}""",
           f"""--task {args.feature}""",
           f"""--framework {args.framework}""" if args.framework is not None else """""",
           f"""{args.output}""",
       ]
       snake_case__					:	Union[str, Any]          =					subprocess.Popen(""" """.join(UpperCAmelCase     )    ,       stdout=subprocess.PIPE    ,       shell=UpperCAmelCase     )
       proc.wait()

       logger.info(
           """The export was done by optimum.exporters.onnx. We recommend using to use this package directly in future, as """
           """transformers.onnx is deprecated, and will be removed in v5. You can find more information here: """
           """https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model."""     )



def 				lowerCAmelCase__	(       UpperCAmelCase     ):






       """simple docstring"""





       snake_case__					:	str          =					args.output if args.output.is_file() else args.output.joinpath("""model.onnx"""     )
       if not args.output.parent.exists():
              args.output.parent.mkdir(parents=UpperCAmelCase     )

       # Allocate the model
       snake_case__					:	Dict          =					FeaturesManager.get_model_from_feature(
           args.feature    ,       args.model    ,       framework=args.framework    ,       cache_dir=args.cache_dir     )

       snake_case__       ,   snake_case__					:	List[Any]          =					FeaturesManager.check_supported_model_or_raise(UpperCAmelCase    ,       feature=args.feature     )
       snake_case__					:	Any          =					model_onnx_config(model.config     )

       if model_kind in ENCODER_DECODER_MODELS:
              snake_case__					:	Optional[int]          =					model.get_encoder()
              snake_case__					:	Union[str, Any]          =					model.get_decoder()

              snake_case__					:	Optional[Any]          =					onnx_config.get_encoder_config(encoder_model.config     )
              snake_case__					:	Any          =					onnx_config.get_decoder_config(
                  encoder_model.config    ,       decoder_model.config    ,       feature=args.feature     )

              if args.opset is None:
                     snake_case__					:	List[Any]          =					max(encoder_onnx_config.default_onnx_opset    ,       decoder_onnx_config.default_onnx_opset     )

              if args.opset < min(encoder_onnx_config.default_onnx_opset    ,       decoder_onnx_config.default_onnx_opset     ):
                     raise ValueError(
                         f"""Opset {args.opset} is not sufficient to export {model_kind}. At least """
                         f""" {min(encoder_onnx_config.default_onnx_opset    ,       decoder_onnx_config.default_onnx_opset     )} is required."""     )

              snake_case__					:	Dict          =					AutoFeatureExtractor.from_pretrained(args.model     )

              snake_case__       ,   snake_case__					:	List[str]          =					export(
                  UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.opset    ,       args.output.parent.joinpath("""encoder_model.onnx"""     )    ,       )

              validate_model_outputs(
                  UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.output.parent.joinpath("""encoder_model.onnx"""     )    ,       UpperCAmelCase    ,       args.atol if args.atol else encoder_onnx_config.atol_for_validation    ,       )

              snake_case__					:	List[Any]          =					AutoTokenizer.from_pretrained(args.model     )

              snake_case__       ,   snake_case__					:	Dict          =					export(
                  UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.opset    ,       args.output.parent.joinpath("""decoder_model.onnx"""     )    ,       )

              validate_model_outputs(
                  UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.output.parent.joinpath("""decoder_model.onnx"""     )    ,       UpperCAmelCase    ,       args.atol if args.atol else decoder_onnx_config.atol_for_validation    ,       )
              logger.info(
                  f"""All good, model saved at: {args.output.parent.joinpath("encoder_model.onnx"     ).as_posix()},"""
                  f""" {args.output.parent.joinpath("decoder_model.onnx"     ).as_posix()}"""     )

       else:
              # Instantiate the appropriate preprocessor
              if args.preprocessor == "auto":
                     snake_case__					:	int          =					get_preprocessor(args.model     )
              elif args.preprocessor == "tokenizer":
                     snake_case__					:	List[str]          =					AutoTokenizer.from_pretrained(args.model     )
              elif args.preprocessor == "image_processor":
                     snake_case__					:	Optional[int]          =					AutoImageProcessor.from_pretrained(args.model     )
              elif args.preprocessor == "feature_extractor":
                     snake_case__					:	Tuple          =					AutoFeatureExtractor.from_pretrained(args.model     )
              elif args.preprocessor == "processor":
                     snake_case__					:	int          =					AutoProcessor.from_pretrained(args.model     )
              else:
                     raise ValueError(f"""Unknown preprocessor type '{args.preprocessor}'"""     )

              # Ensure the requested opset is sufficient
              if args.opset is None:
                     snake_case__					:	Dict          =					onnx_config.default_onnx_opset

              if args.opset < onnx_config.default_onnx_opset:
                     raise ValueError(
                         f"""Opset {args.opset} is not sufficient to export {model_kind}. """
                         f"""At least  {onnx_config.default_onnx_opset} is required."""     )

              snake_case__       ,   snake_case__					:	Dict          =					export(
                  UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.opset    ,       args.output    ,       )

              if args.atol is None:
                     snake_case__					:	int          =					onnx_config.atol_for_validation

              validate_model_outputs(UpperCAmelCase    ,       UpperCAmelCase    ,       UpperCAmelCase    ,       args.output    ,       UpperCAmelCase    ,       args.atol     )
              logger.info(f"""All good, model saved at: {args.output.as_posix()}"""     )
              warnings.warn(
                  """The export was done by transformers.onnx which is deprecated and will be removed in v5. We recommend"""
                  """ using optimum.exporters.onnx in future. You can find more information here:"""
                  """ https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model."""    ,       UpperCAmelCase    ,       )



def 				lowerCAmelCase__	(       ):






       """simple docstring"""





       snake_case__					:	List[str]          =					ArgumentParser("""Hugging Face Transformers ONNX exporter"""     )
       parser.add_argument(
           """-m"""    ,       """--model"""    ,       type=UpperCAmelCase    ,       required=UpperCAmelCase    ,       help="""Model ID on huggingface.co or path on disk to load model from."""     )
       parser.add_argument(
           """--feature"""    ,       default="""default"""    ,       help="""The type of features to export the model with."""    ,       )
       parser.add_argument("""--opset"""    ,       type=UpperCAmelCase    ,       default=UpperCAmelCase    ,       help="""ONNX opset version to export the model with."""     )
       parser.add_argument(
           """--atol"""    ,       type=UpperCAmelCase    ,       default=UpperCAmelCase    ,       help="""Absolute difference tolerance when validating the model."""     )
       parser.add_argument(
           """--framework"""    ,       type=UpperCAmelCase    ,       choices=["""pt""", """tf"""]    ,       default=UpperCAmelCase    ,       help=(
               """The framework to use for the ONNX export."""
               """ If not provided, will attempt to use the local checkpoint's original framework"""
               """ or what is available in the environment."""
           )    ,       )
       parser.add_argument("""output"""    ,       type=UpperCAmelCase    ,       help="""Path indicating where to store generated ONNX model."""     )
       parser.add_argument("""--cache_dir"""    ,       type=UpperCAmelCase    ,       default=UpperCAmelCase    ,       help="""Path indicating where to store cache."""     )
       parser.add_argument(
           """--preprocessor"""    ,       type=UpperCAmelCase    ,       choices=["""auto""", """tokenizer""", """feature_extractor""", """image_processor""", """processor"""]    ,       default="""auto"""    ,       help="""Which type of preprocessor to use. 'auto' tries to automatically detect it."""    ,       )
       parser.add_argument(
           """--export_with_transformers"""    ,       action="""store_true"""    ,       help=(
               """Whether to use transformers.onnx instead of optimum.exporters.onnx to perform the ONNX export. It can be """
               """useful when exporting a model supported in transformers but not in optimum, otherwise it is not """
               """recommended."""
           )    ,       )

       snake_case__					:	str          =					parser.parse_args()
       if args.export_with_transformers or not is_optimum_available():
              export_with_transformers(UpperCAmelCase     )
       else:
              export_with_optimum(UpperCAmelCase     )


if __name__ == "__main__":
     lowerCAmelCase__  =     logging.get_logger('transformers.onnx')  # pylint: disable=invalid-name
     logger.setLevel(logging.INFO)
     main()




