


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


SCREAMING_SNAKE_CASE__				:  Optional[Any]						= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__				:  Optional[int]						= {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__				:  Dict						= {
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE__				:  List[Any]						= {
    "microsoft/layoutlm-base-uncased": 512,
    "microsoft/layoutlm-large-uncased": 512,
}

SCREAMING_SNAKE_CASE__				:  Tuple						= {
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}


class 	A_       (     _UpperCAmelCase   ):





  """simple docstring"""

  lowercase		:	List[str]   =						VOCAB_FILES_NAMES
  lowercase		:	Any   =						PRETRAINED_VOCAB_FILES_MAP
  lowercase		:	Any   =						PRETRAINED_INIT_CONFIGURATION
  lowercase		:	Optional[Any]   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase		:	Union[str, Any]   =						LayoutLMTokenizer
  def __init__(						self  ,					__UpperCAmelCase=None  ,					__UpperCAmelCase=None  ,					__UpperCAmelCase=True  ,					__UpperCAmelCase="[UNK]"  ,					__UpperCAmelCase="[SEP]"  ,					__UpperCAmelCase="[PAD]"  ,					__UpperCAmelCase="[CLS]"  ,					__UpperCAmelCase="[MASK]"  ,					__UpperCAmelCase=True  ,					__UpperCAmelCase=None  ,					**__UpperCAmelCase  ,					)				->					Optional[int]:
     super().__init__(
         __UpperCAmelCase  ,					tokenizer_file=__UpperCAmelCase  ,					do_lower_case=__UpperCAmelCase  ,					unk_token=__UpperCAmelCase  ,					sep_token=__UpperCAmelCase  ,					pad_token=__UpperCAmelCase  ,					cls_token=__UpperCAmelCase  ,					mask_token=__UpperCAmelCase  ,					tokenize_chinese_chars=__UpperCAmelCase  ,					strip_accents=__UpperCAmelCase  ,					**__UpperCAmelCase  ,					)

     a   :	Tuple						=    json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
     if (
         normalizer_state.get('lowercase'  ,					__UpperCAmelCase						) != do_lower_case
         or normalizer_state.get('strip_accents'  ,					__UpperCAmelCase						) != strip_accents
         or normalizer_state.get('handle_chinese_chars'  ,					__UpperCAmelCase						) != tokenize_chinese_chars
     ):
        a   :	int						=    getattr(__UpperCAmelCase  ,					normalizer_state.pop('type'						)						)
        a   :	List[str]						=    do_lower_case
        a   :	List[Any]						=    strip_accents
        a   :	Optional[int]						=    tokenize_chinese_chars
        a   :	int						=    normalizer_class(**__UpperCAmelCase						)

     a   :	Tuple						=    do_lower_case
  def 							lowercase_     (						self  ,					__UpperCAmelCase  ,					__UpperCAmelCase=None						)				->					Dict:
     a   :	List[Any]						=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

     return output
  def 							lowercase_     (						self  ,					__UpperCAmelCase  ,					__UpperCAmelCase = None						)				->					List[int]:
     a   :	Dict						=    [self.sep_token_id]
     a   :	List[str]						=    [self.cls_token_id]
     if token_ids_a is None:
        return len(cls + token_ids_a + sep						) * [0]
     return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






  def 							lowercase_     (						self  ,					__UpperCAmelCase  ,					__UpperCAmelCase = None						)				->					Tuple[str]:
     a   :	Dict						=    self._tokenizer.model.save(__UpperCAmelCase  ,					name=__UpperCAmelCase						)
     return tuple(__UpperCAmelCase						)









"""simple docstring"""






import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata




def        A_					(					UpperCAmelCase__				)     ->     str:
   a   :	Optional[int]						=    min(re.search('^ +'       ,   UpperCAmelCase__				).end() if t.startswith(' '				) else 0 for t in string.splitlines()				)
   return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(UpperCAmelCase__				)]				)


SCREAMING_SNAKE_CASE__				:  Union[str, Any]						= "\\n---\nlanguage:\n- zh\n- en\ntask_ids:\n- sentiment-classification\n---\n# Begin of markdown\n\nSome cool dataset card\n"

SCREAMING_SNAKE_CASE__				:  List[str]						= "\\n---\n---\n# Begin of markdown\n\nSome cool dataset card\n"


SCREAMING_SNAKE_CASE__				:  Dict						= "\\n# Begin of markdown\n\nSome cool dataset card\n"


class 	A_       (     unittest.TestCase   ):





  """simple docstring"""

  def 							lowercase_     (						self						)				->					Any:
     with tempfile.TemporaryDirectory() as tmp_dir:
        a   :	Optional[Any]						=    Path(__UpperCAmelCase						) / 'README.md'
        with open(__UpperCAmelCase  ,					'w+'						) as readme_file:
           readme_file.write(__UpperCAmelCase						)
        a   :	str						=    DatasetMetadata.from_readme(__UpperCAmelCase						)
        self.assertDictEqual(__UpperCAmelCase  ,					{'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}						)

        with open(__UpperCAmelCase  ,					'w+'						) as readme_file:
           readme_file.write(__UpperCAmelCase						)
        a   :	Optional[Any]						=    DatasetMetadata.from_readme(__UpperCAmelCase						)
        self.assertDictEqual(__UpperCAmelCase  ,					{}						)

        with open(__UpperCAmelCase  ,					'w+'						) as readme_file:
           readme_file.write(__UpperCAmelCase						)
        a   :	str						=    DatasetMetadata.from_readme(__UpperCAmelCase						)
        self.assertEqual(__UpperCAmelCase  ,					{}						)






  def 							lowercase_     (						self						)				->					Tuple:
     a   :	List[str]						=    _dedent(
         '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '						)
     assert DatasetMetadata.from_yaml_string(__UpperCAmelCase						)

     a   :	Optional[Any]						=    _dedent(
         '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '						)
     with self.assertRaises(__UpperCAmelCase						):
        DatasetMetadata.from_yaml_string(__UpperCAmelCase						)

     a   :	List[Any]						=    _dedent(
         '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '						)
     assert DatasetMetadata.from_yaml_string(__UpperCAmelCase						)





