
import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
      import importlib.resources as pkg_resources
except ImportError:
      # Try backported to PY<37 `importlib_resources`.
      import importlib_resources as pkg_resources

from . import resources


_A							=       '''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
_A							=       F'''{BASE_REF_URL}/{__file__}'''
_A							=       logging.getLogger(__name__)




def 					lowerCamelCase__					(    a__	:	str  )		->						Tuple[Any, str]:
       UpperCamelCase_											=							pkg_resources.read_text(a__						,   a__  )
       return yaml.safe_load(a__  ), f'''{BASE_REF_URL}/resources/{resource}'''


_A	,    _A							=       load_yaml_resource('''readme_structure.yaml''')

_A							=       [
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
_A							=       Tuple[dict, List[str], List[str]]

class 		lowercase_							:

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        UpperCamelCase_											=							name
        UpperCamelCase_											=							level
        UpperCamelCase_											=							lines
        UpperCamelCase_											=							""""""
        UpperCamelCase_											=							True
        UpperCamelCase_											=							{}
        UpperCamelCase_											=							[]
        UpperCamelCase_											=							[]
        if self.lines is not None:
               self.parse(suppress_parsing_errors=__UpperCamelCase  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        UpperCamelCase_											=							""""""
        UpperCamelCase_											=							[]
        UpperCamelCase_											=							False
        for line in self.lines:
               if line.strip(""" \n"""  ) == "":
                      continue
               elif line.strip(""" \n"""  )[:3] == "```":
                      UpperCamelCase_											=							not code_start
               elif line.split()[0] == self.level + "#" and not code_start:
                      if current_sub_level != "":
                             UpperCamelCase_											=							Section(__UpperCamelCase  ,						self.level + """#"""  ,						__UpperCamelCase  )
                             UpperCamelCase_											=							[]
                      else:
                             if current_lines != []:
                                    self.text += "".join(__UpperCamelCase  ).strip()
                                    if self.text != "" and self.text not in FILLER_TEXT:
                                           UpperCamelCase_											=							False
                                    UpperCamelCase_											=							[]

                      UpperCamelCase_											=							""" """.join(line.split()[1:]  ).strip(""" \n"""  )
               else:
                      current_lines.append(__UpperCamelCase  )
        else:
               if current_sub_level != "":
                      if current_sub_level in self.content:
                             self.parsing_error_list.append(
                                 f'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''  )
                      UpperCamelCase_											=							Section(__UpperCamelCase  ,						self.level + """#"""  ,						__UpperCamelCase  )
               else:
                      if current_lines != []:
                             self.text += "".join(__UpperCamelCase  ).strip()
                             if self.text != "" and self.text not in FILLER_TEXT:
                                    UpperCamelCase_											=							False

        if self.level == "" and not suppress_parsing_errors:
               if self.parsing_error_list != [] or self.parsing_warning_list != []:
                      UpperCamelCase_											=							UpperCamelCase_											=							"""\n""".join("""-\t""" + x for x in self.parsing_error_list + self.parsing_warning_list  )
                      UpperCamelCase_											=							f'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
                      raise ValueError(__UpperCamelCase  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							[]
        UpperCamelCase_											=							[]
        if structure["allow_empty"] is False:
               # If content is expected
               if self.is_empty_text and self.content == {}:
                      # If no content is found, mention it in the error_list
                      error_list.append(f'''Expected some content in section `{self.name}` but it is empty.'''  )

        if structure["allow_empty_text"] is False:
               # If some text is expected
               if self.is_empty_text:
                      # If no text is found, mention it in the error_list
                      error_list.append(
                          f'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''  )
        # Subsections Validation
        if structure["subsections"] is not None:
               # If subsections are expected
               if self.content == {}:
                      # If no subsections are present
                      UpperCamelCase_											=							[subsection["""name"""] for subsection in structure["""subsections"""]]
                      # Mention the expected values in the error_list
                      error_list.append(
                          f'''Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]  )}. Found \'None\'.'''  )
               else:
                      # If some subsections are present
                      UpperCamelCase_											=							[subsection["""name"""] for subsection in structure["""subsections"""]]
                      UpperCamelCase_											=							False
                      for idx, name in enumerate(__UpperCamelCase  ):
                             if name not in self.content:
                                    # If the expected subsection is not present
                                    error_list.append(f'''Section `{self.name}` is missing subsection: `{name}`.'''  )
                                    UpperCamelCase_											=							True
                             else:
                                    # If the subsection is present, validate subsection, return the result
                                    # and concat the errors from subsection to section error_list

                                    # Skip sublevel validation if current level is `###`
                                    if self.level == "###":
                                           continue
                                    else:
                                           UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_											=							self.content[name].validate(
                                               structure["""subsections"""][idx]  )
                                    error_list += subsec_error_list
                                    warning_list += subsec_warning_list

                      if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                             for name in self.content:
                                    if name not in structure_names:
                                           # If an extra subsection is present
                                           warning_list.append(
                                               f'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''  )
        if error_list:
               # If there are errors, do not return the dictionary as it is invalid
               return {}, error_list, warning_list
        else:
               return self.to_dict(), error_list, warning_list

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        return {
            "name": self.name,
            "text": self.text,
            "is_empty_text": self.is_empty_text,
            "subsections": [value.to_dict() for value in self.content.values()],
        }



class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):  # Level 0

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        super().__init__(name=__UpperCamelCase  ,						level=""""""  )  # Not using lines here as we need to use a child class parse
        UpperCamelCase_											=							structure
        UpperCamelCase_											=							-2
        UpperCamelCase_											=							0
        UpperCamelCase_											=							lines
        if self.lines is not None:
               self.parse(suppress_parsing_errors=__UpperCamelCase  )

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        if self.structure is None:
               UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_											=							self._validate(__UpperCamelCase  )
        else:
               UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_											=							self._validate(self.structure  )
        if error_list != [] or warning_list != []:
               UpperCamelCase_											=							"""\n""".join(["""-\t""" + x for x in error_list + warning_list]  )
               UpperCamelCase_											=							f'''The following issues were found for the README at `{self.name}`:\n''' + errors
               raise ValueError(__UpperCamelCase  )

 @classmethod
 def lowerCamelCase_					(   cls  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        with open(__UpperCamelCase  ,						encoding="""utf-8"""  ) as f:
               UpperCamelCase_											=							f.readlines()
        return cls(__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						suppress_parsing_errors=__UpperCamelCase  )

 @classmethod
 def lowerCamelCase_					(   cls  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = "root"  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        UpperCamelCase_											=							string.split("""\n"""  )
        return cls(__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						suppress_parsing_errors=__UpperCamelCase  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase = False  ):




        """simple docstring"""


        UpperCamelCase_											=							0

        for line in self.lines:
               self.yaml_tags_line_count += 1
               if line.strip(""" \n"""  ) == "---":
                      self.tag_count += 1
                      if self.tag_count == 2:
                             break
               line_count += 1
        if self.tag_count == 2:
               UpperCamelCase_											=							self.lines[line_count + 1 :]  # Get the last + 1 th item.
        else:
               UpperCamelCase_											=							self.lines[self.tag_count :]
        super().parse(suppress_parsing_errors=__UpperCamelCase  )

 def __str__(   self  ):




        """simple docstring"""


        return str(self.to_dict()  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							[]
        UpperCamelCase_											=							[]
        if self.yaml_tags_line_count == 0:
               warning_list.append("""Empty YAML markers are present in the README."""  )
        elif self.tag_count == 0:
               warning_list.append("""No YAML markers are present in the README."""  )
        elif self.tag_count == 1:
               warning_list.append("""Only the start of YAML tags present in the README."""  )
        # Check how many first level sections are present.
        UpperCamelCase_											=							len(self.content.keys()  )
        if num_first_level_keys > 1:
               # If more than one, add to the error list, continue
               error_list.append(
                   f'''The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()  )]  )}. Only one heading is expected. Skipping further validation for this README.'''  )
        elif num_first_level_keys < 1:
               # If less than one, append error.
               error_list.append(
                   """The README has no first-level headings. One heading is expected. Skipping further validation for this README."""  )

        else:
               # If one exactly
               UpperCamelCase_											=							list(self.content.keys()  )[0]  # Get the key
               if start_key.startswith("""Dataset Card for"""  ):  # Check correct start
                      # If the starting is correct, validate all the sections
                      UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_											=							self.content[start_key].validate(
                          readme_structure["""subsections"""][0]  )
                      error_list += sec_error_list
                      warning_list += sec_warning_list
               else:
                      # If not found, append error
                      error_list.append(
                          """No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."""  )
        if error_list:
               # If there are errors, do not return the dictionary as it is invalid
               return {}, error_list, warning_list
        else:
               return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
      from argparse import ArgumentParser

      _A							=       ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
      ap.add_argument('''readme_filepath''')
      _A							=       ap.parse_args()
      _A							=       Path(args.readme_filepath)
      _A							=       ReadMe.from_readme(readme_filepath)





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = None  ,						__UpperCamelCase = None  ,						__UpperCamelCase = False  ,						__UpperCamelCase = False  ,						__UpperCamelCase = None  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        super().__init__(
            __UpperCamelCase  ,						split=__UpperCamelCase  ,						features=__UpperCamelCase  ,						cache_dir=__UpperCamelCase  ,						keep_in_memory=__UpperCamelCase  ,						streaming=__UpperCamelCase  ,						num_proc=__UpperCamelCase  ,						**__UpperCamelCase  ,						)
        UpperCamelCase_											=							path_or_paths if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else {self.split: path_or_paths}
        UpperCamelCase_											=							Csv(
            cache_dir=__UpperCamelCase  ,						data_files=__UpperCamelCase  ,						features=__UpperCamelCase  ,						**__UpperCamelCase  ,						)

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        if self.streaming:
               UpperCamelCase_											=							self.builder.as_streaming_dataset(split=self.split  )
        # Build regular (map-style) dataset
        else:
               UpperCamelCase_											=							None
               UpperCamelCase_											=							None
               UpperCamelCase_											=							None
               UpperCamelCase_											=							None

               self.builder.download_and_prepare(
                   download_config=__UpperCamelCase  ,						download_mode=__UpperCamelCase  ,						verification_mode=__UpperCamelCase  ,						base_path=__UpperCamelCase  ,						num_proc=self.num_proc  ,						)
               UpperCamelCase_											=							self.builder.as_dataset(
                   split=self.split  ,						verification_mode=__UpperCamelCase  ,						in_memory=self.keep_in_memory  )
        return dataset



class 		lowercase_							:

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ,						__UpperCamelCase = None  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        if num_proc is not None and num_proc <= 0:
               raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''  )

        UpperCamelCase_											=							dataset
        UpperCamelCase_											=							path_or_buf
        UpperCamelCase_											=							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        UpperCamelCase_											=							num_proc
        UpperCamelCase_											=							"""utf-8"""
        UpperCamelCase_											=							to_csv_kwargs

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        UpperCamelCase_											=							self.to_csv_kwargs.pop("""path_or_buf"""  ,						__UpperCamelCase  )
        UpperCamelCase_											=							self.to_csv_kwargs.pop("""header"""  ,						__UpperCamelCase  )
        UpperCamelCase_											=							self.to_csv_kwargs.pop("""index"""  ,						__UpperCamelCase  )

        if isinstance(self.path_or_buf  ,						(str, bytes, os.PathLike)  ):
               with open(self.path_or_buf  ,						"""wb+"""  ) as buffer:
                      UpperCamelCase_											=							self._write(file_obj=__UpperCamelCase  ,						header=__UpperCamelCase  ,						index=__UpperCamelCase  ,						**self.to_csv_kwargs  )
        else:
               UpperCamelCase_											=							self._write(file_obj=self.path_or_buf  ,						header=__UpperCamelCase  ,						index=__UpperCamelCase  ,						**self.to_csv_kwargs  )
        return written

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_											=							args

        UpperCamelCase_											=							query_table(
            table=self.dataset.data  ,						key=slice(__UpperCamelCase  ,						offset + self.batch_size  )  ,						indices=self.dataset._indices  ,						)
        UpperCamelCase_											=							batch.to_pandas().to_csv(
            path_or_buf=__UpperCamelCase  ,						header=header if (offset == 0) else False  ,						index=__UpperCamelCase  ,						**__UpperCamelCase  )
        return csv_str.encode(self.encoding  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						**__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							0

        if self.num_proc is None or self.num_proc == 1:
               for offset in logging.tqdm(
                   range(0  ,						len(self.dataset  )  ,						self.batch_size  )  ,						unit="""ba"""  ,						disable=not logging.is_progress_bar_enabled()  ,						desc="""Creating CSV from Arrow format"""  ,						):
                      UpperCamelCase_											=							self._batch_csv((offset, header, index, to_csv_kwargs)  )
                      written += file_obj.write(__UpperCamelCase  )

        else:
               UpperCamelCase_							,					UpperCamelCase_											=							len(self.dataset  ), self.batch_size
               with multiprocessing.Pool(self.num_proc  ) as pool:
                      for csv_str in logging.tqdm(
                          pool.imap(
                              self._batch_csv  ,						[(offset, header, index, to_csv_kwargs) for offset in range(0  ,						__UpperCamelCase  ,						__UpperCamelCase  )]  ,						)  ,						total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size  ,						unit="""ba"""  ,						disable=not logging.is_progress_bar_enabled()  ,						desc="""Creating CSV from Arrow format"""  ,						):
                             written += file_obj.write(__UpperCamelCase  )

        return written



