






"""simple docstring"""



import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
    import importlib.resources as pkg_resources
except ImportError:
    # Try backported to PY<37 `importlib_resources`.
    import importlib_resources as pkg_resources

from . import resources


_UpperCamelCase						:    str		=						'https://github.com/huggingface/datasets/tree/main/src/datasets/utils'
_UpperCamelCase						:    Optional[int]		=						F'''{BASE_REF_URL}/{__file__}'''
_UpperCamelCase						:    Tuple		=						logging.getLogger(__name__)
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    str       ):





  '''simple docstring'''





  lowercase				       =			pkg_resources.read_text(__snake_case						,   __snake_case       )
  return yaml.safe_load(__snake_case       ), f'{BASE_REF_URL}/resources/{resource}'


_UpperCamelCase,						_UpperCamelCase						:    List[str]		=						load_yaml_resource('readme_structure.yaml')

_UpperCamelCase						:    List[str]		=						[
    '[Needs More Information]',
    '[More Information Needed]',
    '(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)',
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCamelCase						:    List[Any]		=						Tuple[dict, List[str], List[str]]

class 			a	:







    def __init__(     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase = None				,						_lowerCamelCase = False			):
      lowercase				       =			name
      lowercase				       =			level
      lowercase				       =			lines
      lowercase				       =			''
      lowercase				       =			True
      lowercase				       =			{}
      lowercase				       =			[]
      lowercase				       =			[]
      if self.lines is not None:
        self.parse(suppress_parsing_errors=_lowerCamelCase			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase = False			):
      lowercase				       =			''
      lowercase				       =			[]
      lowercase				       =			False
      for line in self.lines:
        if line.strip(' \n'			) == "":
          continue
        elif line.strip(' \n'			)[:3] == "```":
          lowercase				       =			not code_start
        elif line.split()[0] == self.level + "#" and not code_start:
          if current_sub_level != "":
            lowercase				       =			Section(_lowerCamelCase				,						self.level + '#'				,						_lowerCamelCase			)
            lowercase				       =			[]
          else:
            if current_lines != []:
              self.text += "".join(_lowerCamelCase			).strip()
              if self.text != "" and self.text not in FILLER_TEXT:
                lowercase				       =			False
              lowercase				       =			[]

          lowercase				       =			' '.join(line.split()[1:]			).strip(' \n'			)
        else:
          current_lines.append(_lowerCamelCase			)
      else:
        if current_sub_level != "":
          if current_sub_level in self.content:
            self.parsing_error_list.append(
                F'Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'			)
          lowercase				       =			Section(_lowerCamelCase				,						self.level + '#'				,						_lowerCamelCase			)
        else:
          if current_lines != []:
            self.text += "".join(_lowerCamelCase			).strip()
            if self.text != "" and self.text not in FILLER_TEXT:
              lowercase				       =			False

      if self.level == "" and not suppress_parsing_errors:
        if self.parsing_error_list != [] or self.parsing_warning_list != []:
          lowercase				       =			lowercase				       =			'\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list			)
          lowercase				       =			F'The following issues were found while parsing the README at `{self.name}`:\n' + errors
          raise ValueError(_lowerCamelCase			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			[]
      lowercase				       =			[]
      if structure["allow_empty"] is False:
        # If content is expected
        if self.is_empty_text and self.content == {}:
          # If no content is found, mention it in the error_list
          error_list.append(F'Expected some content in section `{self.name}` but it is empty.'			)

      if structure["allow_empty_text"] is False:
        # If some text is expected
        if self.is_empty_text:
          # If no text is found, mention it in the error_list
          error_list.append(
              F'Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'			)
        # Subsections Validation
      if structure["subsections"] is not None:
        # If subsections are expected
        if self.content == {}:
          # If no subsections are present
          lowercase				       =			[subsection['name'] for subsection in structure['subsections']]
          # Mention the expected values in the error_list
          error_list.append(
              F'Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]			)}. Found \'None\'.'			)
        else:
          # If some subsections are present
          lowercase				       =			[subsection['name'] for subsection in structure['subsections']]
          lowercase				       =			False
          for idx, name in enumerate(_lowerCamelCase			):
            if name not in self.content:
              # If the expected subsection is not present
              error_list.append(F'Section `{self.name}` is missing subsection: `{name}`.'			)
              lowercase				       =			True
            else:
              # If the subsection is present, validate subsection, return the result
              # and concat the errors from subsection to section error_list

              # Skip sublevel validation if current level is `###`
              if self.level == "###":
                continue
              else:
                lowercase       ,	lowercase       ,	lowercase				       =			self.content[name].validate(
                    structure['subsections'][idx]			)
              error_list += subsec_error_list
              warning_list += subsec_warning_list

          if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
            for name in self.content:
              if name not in structure_names:
                # If an extra subsection is present
                warning_list.append(
                    F'`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'			)
      if error_list:
        # If there are errors, do not return the dictionary as it is invalid
        return {}, error_list, warning_list
      else:
        return self.to_dict(), error_list, warning_list







    def      UpperCamelCase_ (     self			):
      return {
          "name": self.name,
          "text": self.text,
          "is_empty_text": self.is_empty_text,
          "subsections": [value.to_dict() for value in self.content.values()],
      }


class 			a	(  a_      ):  # Level 0







    def __init__(     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase = None				,						_lowerCamelCase = False			):
      super().__init__(name=_lowerCamelCase				,						level=''			)  # Not using lines here as we need to use a child class parse
      lowercase				       =			structure
      lowercase				       =			-2
      lowercase				       =			0
      lowercase				       =			lines
      if self.lines is not None:
        self.parse(suppress_parsing_errors=_lowerCamelCase			)







    def      UpperCamelCase_ (     self			):
      if self.structure is None:
        lowercase       ,	lowercase       ,	lowercase				       =			self._validate(_lowerCamelCase			)
      else:
        lowercase       ,	lowercase       ,	lowercase				       =			self._validate(self.structure			)
      if error_list != [] or warning_list != []:
        lowercase				       =			'\n'.join(['-\t' + x for x in error_list + warning_list]			)
        lowercase				       =			F'The following issues were found for the README at `{self.name}`:\n' + errors
        raise ValueError(_lowerCamelCase			)







    @classmethod
    def      UpperCamelCase_ (     cls				,						_lowerCamelCase				,						_lowerCamelCase = None				,						_lowerCamelCase = False			):
      with open(_lowerCamelCase				,						encoding='utf-8'			) as f:
        lowercase				       =			f.readlines()
      return cls(_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						suppress_parsing_errors=_lowerCamelCase			)







    @classmethod
    def      UpperCamelCase_ (     cls				,						_lowerCamelCase				,						_lowerCamelCase = None				,						_lowerCamelCase = "root"				,						_lowerCamelCase = False			):
      lowercase				       =			string.split('\n'			)
      return cls(_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						suppress_parsing_errors=_lowerCamelCase			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase = False			):
      # Skip Tags
      lowercase				       =			0

      for line in self.lines:
        self.yaml_tags_line_count += 1
        if line.strip(' \n'			) == "---":
          self.tag_count += 1
          if self.tag_count == 2:
            break
        line_count += 1
      if self.tag_count == 2:
        lowercase				       =			self.lines[line_count + 1 :]  # Get the last + 1 th item.
      else:
        lowercase				       =			self.lines[self.tag_count :]
      super().parse(suppress_parsing_errors=_lowerCamelCase			)







    def __str__(     self			):
      return str(self.to_dict()			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			[]
      lowercase				       =			[]
      if self.yaml_tags_line_count == 0:
        warning_list.append('Empty YAML markers are present in the README.'			)
      elif self.tag_count == 0:
        warning_list.append('No YAML markers are present in the README.'			)
      elif self.tag_count == 1:
        warning_list.append('Only the start of YAML tags present in the README.'			)
      # Check how many first level sections are present.
      lowercase				       =			len(self.content.keys()			)
      if num_first_level_keys > 1:
        # If more than one, add to the error list, continue
        error_list.append(
            F'The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()			)]			)}. Only one heading is expected. Skipping further validation for this README.'			)
      elif num_first_level_keys < 1:
        # If less than one, append error.
        error_list.append(
            'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'			)

      else:
        # If one exactly
        lowercase				       =			list(self.content.keys()			)[0]  # Get the key
        if start_key.startswith('Dataset Card for'			):  # Check correct start
          # If the starting is correct, validate all the sections
          lowercase       ,	lowercase       ,	lowercase				       =			self.content[start_key].validate(
              readme_structure['subsections'][0]			)
          error_list += sec_error_list
          warning_list += sec_warning_list
        else:
          # If not found, append error
          error_list.append(
              'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'			)
      if error_list:
        # If there are errors, do not return the dictionary as it is invalid
        return {}, error_list, warning_list
      else:
        return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
    from argparse import ArgumentParser

    _UpperCamelCase						:    Optional[int]		=						ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
    ap.add_argument('readme_filepath')
    _UpperCamelCase						:    Tuple		=						ap.parse_args()
    _UpperCamelCase						:    Union[str, Any]		=						Path(args.readme_filepath)
    _UpperCamelCase						:    Union[str, Any]		=						ReadMe.from_readme(readme_filepath)










"""simple docstring"""



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
    import torch

    from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer

@require_torch
class 			a	(  unittest.TestCase      ):







    @slow
    def      UpperCamelCase_ (     self			):
      lowercase				       =			XLMProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'			)
      model.to(_lowerCamelCase			)

      # encoder-decoder outputs
      lowercase				       =			torch.tensor([[1_7, 9_6_2_0_8, 1_0_3_4_7_1, 2]]			).to(_lowerCamelCase			)
      lowercase				       =			torch.tensor(
          [[2, 2_5_0, 9_9_5_3, 3_4, 6_9_4_8_9, 1_6_2_0, 3_2, 1_1_8_4_2_4, 6_2_4, 2_1_0, 1_0_5, 2_9_1_3, 1_0_3_2, 3_5_1]]			).to(_lowerCamelCase			)
      lowercase				       =			model(
          input_ids=_lowerCamelCase				,						attention_mask=_lowerCamelCase				,						encoder_outputs=_lowerCamelCase				,						decoder_input_ids=_lowerCamelCase			)
      lowercase				       =			output[0]
      lowercase				       =			torch.Size((1, 1_4, 2_5_0_0_1_2)			)
      self.assertEqual(output_predited_logis.shape				,						_lowerCamelCase			)
      lowercase				       =			torch.tensor(
          [[[-6.3_9_8_6, -8.2_3_9_1, 1_2.5_1_8_9], [-6.3_2_8_9, -8.0_8_6_4, 1_2.6_2_1_1], [-6.2_4_1_8, -8.0_4_4_5, 1_2.7_9_6_8]]]			).to(_lowerCamelCase			)
      self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]				,						_lowerCamelCase				,						atol=1e-4			)			)

      # encoder outputs
      lowercase				       =			model.prophetnet.encoder(_lowerCamelCase			)[0]
      lowercase				       =			torch.tensor(
          [[[-1.4_2_6_0, -0.7_6_2_8, 0.8_4_5_3], [-1.4_7_1_9, -0.1_3_9_1, 0.7_8_0_7], [-1.7_6_7_8, 0.0_1_1_4, 0.4_6_4_6]]]			).to(_lowerCamelCase			)
      lowercase				       =			torch.Size((1, 4, 1_0_2_4)			)
      self.assertEqual(encoder_outputs.shape				,						_lowerCamelCase			)
      self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]				,						_lowerCamelCase				,						atol=1e-4			)			)

      # decoder outputs
      lowercase				       =			model.prophetnet.decoder(
          _lowerCamelCase				,						encoder_hidden_states=_lowerCamelCase				,						)
      lowercase				       =			decoder_outputs[1].view(1				,						model.config.ngram				,						1_4				,						-1			)
      lowercase				       =			model.lm_head(_lowerCamelCase			)
      lowercase				       =			predicting_streams_logits[:, 0]
      self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]				,						_lowerCamelCase				,						atol=1e-4			)			)







    @slow
    def      UpperCamelCase_ (     self			):
      lowercase				       =			XLMProphetNetForConditionalGeneration.from_pretrained(
          'microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'			)
      model.to(_lowerCamelCase			)

      lowercase				       =			torch.tensor([[1_7, 9_6_2_0_8, 1_0_3_4_7_1, 2]]			).to(_lowerCamelCase			)
      lowercase				       =			torch.tensor(
          [[2, 2_5_0, 9_9_5_3, 3_4, 6_9_4_8_9, 1_6_2_0, 3_2, 1_1_8_4_2_4, 6_2_4, 2_1_0, 1_0_5, 2_9_1_3, 1_0_3_2, 3_5_1]]			).to(_lowerCamelCase			)
      lowercase				       =			model(
          input_ids=_lowerCamelCase				,						attention_mask=_lowerCamelCase				,						encoder_outputs=_lowerCamelCase				,						decoder_input_ids=_lowerCamelCase			)
      lowercase				       =			output[0]
      lowercase				       =			torch.Size((1, 1_4, 2_5_0_0_1_2)			)
      self.assertEqual(output_predited_logis.shape				,						_lowerCamelCase			)
      # compare the actual values for a slice.
      lowercase				       =			torch.tensor(
          [[[-9.2_2_5_3, -9.7_1_7_3, -6.3_5_2_9], [-7.6_7_0_1, -9.0_1_4_5, -1.9_3_8_2], [-8.0_1_9_5, -7.0_0_0_4, -0.1_5_2_3]]]			).to(_lowerCamelCase			)

      self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]				,						_lowerCamelCase				,						atol=1e-4			)			)







    @slow
    def      UpperCamelCase_ (     self			):
      lowercase				       =			XLMProphetNetForConditionalGeneration.from_pretrained(
          'microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'			)
      model.to(_lowerCamelCase			)
      lowercase				       =			5_1_2

      lowercase				       =			XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'			)

      lowercase				       =			(
          'Microsoft Corporation intends to officially end free support for the Windows 7 operating system after'
          ' January 14, 2020, according to the official portal of the organization. From that day, users of this'
          ' system will not be able to receive security updates, which could make their computers vulnerable to'
          ' cyber attacks.'
      )
      lowercase				       =			(
          'орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7'
          ' после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи'
          ' этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми'
          ' к кибератакам.'
      )
      lowercase				       =			(
          '根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。'
      )

      lowercase				       =			tokenizer(
          [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]				,						padding=_lowerCamelCase				,						max_length=2_5_5				,						return_tensors='pt'			).input_ids
      lowercase				       =			input_ids.to(_lowerCamelCase			)

      lowercase				       =			model.generate(
          _lowerCamelCase				,						num_beams=1_0				,						length_penalty=1.0				,						no_repeat_ngram_size=3				,						early_stopping=_lowerCamelCase			)
      lowercase				       =			[tokenizer.decode(_lowerCamelCase				,						skip_special_tokens=_lowerCamelCase			) for g in summary_ids]
      lowercase				       =			'Microsoft to end Windows 7 free support after January 14, 2020'
      lowercase				       =			'Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года'
      lowercase				       =			'微软打算终止对Windows 7操作系统的免费支持'
      self.assertListEqual(
          [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]				,						_lowerCamelCase				,						)

      lowercase				       =			model.generate(
          _lowerCamelCase				,						num_beams=1				,						length_penalty=1.0				,						no_repeat_ngram_size=3				,						early_stopping=_lowerCamelCase			)
      lowercase				       =			[
          tokenizer.convert_ids_to_tokens(_lowerCamelCase				,						skip_special_tokens=_lowerCamelCase			) for g in summary_ids_beama
      ]
      lowercase				       =			'▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7'.split(' '			)
      lowercase				       =			'▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года'.split(
          ' '			)
      lowercase				       =			'微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持'.split(' '			)
      self.assertListEqual(
          [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]				,						_lowerCamelCase				,						)


