





"""simple docstring"""


import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
   import importlib.resources as pkg_resources
except ImportError:
   # Try backported to PY<37 `importlib_resources`.
   import importlib_resources as pkg_resources

from . import resources


a__       :  int       		=					"""https://github.com/huggingface/datasets/tree/main/src/datasets/utils"""
a__       :  List[str]       		=					f'''{BASE_REF_URL}/{__file__}'''
a__       :  Any       		=					logging.getLogger(__name__)





def 				a__      (	_UpperCAmelCase						)     ->	Tuple[Any, str]:

 '''simple docstring'''






 snake_case__     				=					pkg_resources.read_text(_UpperCAmelCase						,  _UpperCAmelCase						)
 return yaml.safe_load(_UpperCAmelCase						), f"{BASE_REF_URL}/resources/{resource}"


a__		,			a__       :  Tuple       		=					load_yaml_resource("""readme_structure.yaml""")

a__       :  Dict       		=					[
    """[Needs More Information]""",
    """[More Information Needed]""",
    """(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)""",
]

# Dictionary representation of section/readme, error_list, warning_list
a__       :  Any       		=					Tuple[dict, List[str], List[str]]



class   lowercase_					:

       def __init__(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE = None ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        snake_case__     				=					name
        snake_case__     				=					level
        snake_case__     				=					lines
        snake_case__     				=					""
        snake_case__     				=					True
        snake_case__     				=					{}
        snake_case__     				=					[]
        snake_case__     				=					[]
        if self.lines is not None:
         self.parse(suppress_parsing_errors=__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        snake_case__     				=					""
        snake_case__     				=					[]
        snake_case__     				=					False
        for line in self.lines:
         if line.strip(" \n"  ) == "":
          continue
         elif line.strip(" \n"  )[:3] == "```":
          snake_case__     				=					not code_start
         elif line.split()[0] == self.level + "#" and not code_start:
          if current_sub_level != "":
           snake_case__     				=					Section(__SCREAMING_SNAKE_CASE ,       self.level + "#" ,       __SCREAMING_SNAKE_CASE  )
           snake_case__     				=					[]
          else:
           if current_lines != []:
            self.text += "".join(__SCREAMING_SNAKE_CASE  ).strip()
            if self.text != "" and self.text not in FILLER_TEXT:
             snake_case__     				=					False
            snake_case__     				=					[]

          snake_case__     				=					" ".join(line.split()[1:]  ).strip(" \n"  )
         else:
          current_lines.append(__SCREAMING_SNAKE_CASE  )
        else:
         if current_sub_level != "":
          if current_sub_level in self.content:
           self.parsing_error_list.append(
               F"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."  )
          snake_case__     				=					Section(__SCREAMING_SNAKE_CASE ,       self.level + "#" ,       __SCREAMING_SNAKE_CASE  )
         else:
          if current_lines != []:
           self.text += "".join(__SCREAMING_SNAKE_CASE  ).strip()
           if self.text != "" and self.text not in FILLER_TEXT:
            snake_case__     				=					False

        if self.level == "" and not suppress_parsing_errors:
         if self.parsing_error_list != [] or self.parsing_warning_list != []:
          snake_case__     				=					snake_case__     				=					"\n".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list  )
          snake_case__     				=					F"The following issues were found while parsing the README at `{self.name}`:\n" + errors
          raise ValueError(__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        snake_case__     				=					[]
        snake_case__     				=					[]
        if structure["allow_empty"] is False:
         # If content is expected
         if self.is_empty_text and self.content == {}:
          # If no content is found, mention it in the error_list
          error_list.append(F"Expected some content in section `{self.name}` but it is empty."  )

        if structure["allow_empty_text"] is False:
         # If some text is expected
         if self.is_empty_text:
          # If no text is found, mention it in the error_list
          error_list.append(
              F"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."  )
        # Subsections Validation
        if structure["subsections"] is not None:
         # If subsections are expected
         if self.content == {}:
          # If no subsections are present
          snake_case__     				=					[subsection["name"] for subsection in structure["subsections"]]
          # Mention the expected values in the error_list
          error_list.append(
              F"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]  )}. Found 'None'."  )
         else:
          # If some subsections are present
          snake_case__     				=					[subsection["name"] for subsection in structure["subsections"]]
          snake_case__     				=					False
          for idx, name in enumerate(__SCREAMING_SNAKE_CASE  ):
           if name not in self.content:
            # If the expected subsection is not present
            error_list.append(F"Section `{self.name}` is missing subsection: `{name}`."  )
            snake_case__     				=					True
           else:
            # If the subsection is present, validate subsection, return the result
            # and concat the errors from subsection to section error_list

            # Skip sublevel validation if current level is `###`
            if self.level == "###":
             continue
            else:
             snake_case__				, snake_case__				, snake_case__     				=					self.content[name].validate(
                 structure["subsections"][idx]  )
            error_list += subsec_error_list
            warning_list += subsec_warning_list

          if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
           for name in self.content:
            if name not in structure_names:
             # If an extra subsection is present
             warning_list.append(
                 F"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."  )
        if error_list:
         # If there are errors, do not return the dictionary as it is invalid
         return {}, error_list, warning_list
        else:
         return self.to_dict(), error_list, warning_list

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        return {
            "name": self.name,
            "text": self.text,
            "is_empty_text": self.is_empty_text,
            "subsections": [value.to_dict() for value in self.content.values()],
        }



class   lowercase_					(  __lowerCAmelCase				):  # Level 0

       def __init__(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE = None ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        super().__init__(name=__SCREAMING_SNAKE_CASE ,       level=""  )  # Not using lines here as we need to use a child class parse
        snake_case__     				=					structure
        snake_case__     				=					-2
        snake_case__     				=					0
        snake_case__     				=					lines
        if self.lines is not None:
         self.parse(suppress_parsing_errors=__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        if self.structure is None:
         snake_case__				, snake_case__				, snake_case__     				=					self._validate(__SCREAMING_SNAKE_CASE  )
        else:
         snake_case__				, snake_case__				, snake_case__     				=					self._validate(self.structure  )
        if error_list != [] or warning_list != []:
         snake_case__     				=					"\n".join(["-\t" + x for x in error_list + warning_list]  )
         snake_case__     				=					F"The following issues were found for the README at `{self.name}`:\n" + errors
         raise ValueError(__SCREAMING_SNAKE_CASE  )

       @classmethod
       def 		_SCREAMING_SNAKE_CASE(      cls ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE = None ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        with open(__SCREAMING_SNAKE_CASE ,       encoding="utf-8"  ) as f:
         snake_case__     				=					f.readlines()
        return cls(__SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       suppress_parsing_errors=__SCREAMING_SNAKE_CASE  )

       @classmethod
       def 		_SCREAMING_SNAKE_CASE(      cls ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE = None ,       __SCREAMING_SNAKE_CASE = "root" ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        snake_case__     				=					string.split("\n"  )
        return cls(__SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       suppress_parsing_errors=__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE = False  ):






        '''simple docstring'''

        snake_case__     				=					0

        for line in self.lines:
         self.yaml_tags_line_count += 1
         if line.strip(" \n"  ) == "---":
          self.tag_count += 1
          if self.tag_count == 2:
           break
         line_count += 1
        if self.tag_count == 2:
         snake_case__     				=					self.lines[line_count + 1 :]  # Get the last + 1 th item.
        else:
         snake_case__     				=					self.lines[self.tag_count :]
        super().parse(suppress_parsing_errors=__SCREAMING_SNAKE_CASE  )

       def __str__(      self  ):






        '''simple docstring'''

        return str(self.to_dict()  )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE  ):






        '''simple docstring'''

        snake_case__     				=					[]
        snake_case__     				=					[]
        if self.yaml_tags_line_count == 0:
         warning_list.append("Empty YAML markers are present in the README."  )
        elif self.tag_count == 0:
         warning_list.append("No YAML markers are present in the README."  )
        elif self.tag_count == 1:
         warning_list.append("Only the start of YAML tags present in the README."  )
        # Check how many first level sections are present.
        snake_case__     				=					len(self.content.keys()  )
        if num_first_level_keys > 1:
         # If more than one, add to the error list, continue
         error_list.append(
             F"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()  )]  )}. Only one heading is expected. Skipping further validation for this README."  )
        elif num_first_level_keys < 1:
         # If less than one, append error.
         error_list.append(
             "The README has no first-level headings. One heading is expected. Skipping further validation for this README."  )

        else:
         # If one exactly
         snake_case__     				=					list(self.content.keys()  )[0]  # Get the key
         if start_key.startswith("Dataset Card for"  ):  # Check correct start
          # If the starting is correct, validate all the sections
          snake_case__				, snake_case__				, snake_case__     				=					self.content[start_key].validate(
              readme_structure["subsections"][0]  )
          error_list += sec_error_list
          warning_list += sec_warning_list
         else:
          # If not found, append error
          error_list.append(
              "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."  )
        if error_list:
         # If there are errors, do not return the dictionary as it is invalid
         return {}, error_list, warning_list
        else:
         return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
   from argparse import ArgumentParser

   a__       :  Union[str, Any]       		=					ArgumentParser(usage="""Validate the content (excluding YAML tags) of a README.md file.""")
   ap.add_argument("""readme_filepath""")
   a__       :  List[Any]       		=					ap.parse_args()
   a__       :  Optional[int]       		=					Path(args.readme_filepath)
   a__       :  Union[str, Any]       		=					ReadMe.from_readme(readme_filepath)








"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a__       :  List[Any]       		=					{
    """configuration_realm""": ["""REALM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RealmConfig"""],
    """tokenization_realm""": ["""RealmTokenizer"""],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   a__       :  Union[str, Any]       		=					["""RealmTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   a__       :  Dict       		=					[
       """REALM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """RealmEmbedder""",
       """RealmForOpenQA""",
       """RealmKnowledgeAugEncoder""",
       """RealmPreTrainedModel""",
       """RealmReader""",
       """RealmScorer""",
       """load_tf_weights_in_realm""",
   ]
   a__       :  Any       		=					["""RealmRetriever"""]


if TYPE_CHECKING:
   from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
   from .tokenization_realm import RealmTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_realm import RealmTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_realm import (
          REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
          RealmEmbedder,
          RealmForOpenQA,
          RealmKnowledgeAugEncoder,
          RealmPreTrainedModel,
          RealmReader,
          RealmScorer,
          load_tf_weights_in_realm,
      )
      from .retrieval_realm import RealmRetriever


else:
   import sys

   a__       :  Any       		=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

