



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class   UpperCAmelCase_    (       a   ,					unittest.TestCase):
							lowerCamelCase__   			=		MobileBertTokenizer
							lowerCamelCase__   			=		MobileBertTokenizerFast
							lowerCamelCase__   			=		True
							lowerCamelCase__   			=		True
							lowerCamelCase__   			=		filter_non_english
							lowerCamelCase__   			=		'google/mobilebert-uncased'







							def    snake_case__      (   self):

														'''simple docstring'''

														super().setUp()

														_lowerCAmelCase     :							Tuple    =    [
														    "[UNK]",
														    "[CLS]",
														    "[SEP]",
														    "[PAD]",
														    "[MASK]",
														    "want",
														    "##want",
														    "##ed",
														    "wa",
														    "un",
														    "runn",
														    "##ing",
														    ",",
														    "low",
														    "lowest",
														]
														_lowerCAmelCase     :							List[str]    =    os.path.join(self.tmpdirname,			VOCAB_FILES_NAMES["vocab_file"])
														with open(self.vocab_file,			"w",			encoding="utf-8") as vocab_writer:
																					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

														_lowerCAmelCase     :							List[Any]    =    [
														    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
														    for tokenizer_def in self.tokenizers_list
														]







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							str    =    "UNwant\u00E9d,running"
														_lowerCAmelCase     :							Optional[Any]    =    "unwanted, running"
														return input_text, output_text







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    self.tokenizer_class(self.vocab_file)

														_lowerCAmelCase     :							int    =    tokenizer.tokenize("UNwant\u00E9d,running")
														self.assertListEqual(__a,			["un", "##want", "##ed", ",", "runn", "##ing"])
														self.assertListEqual(tokenizer.convert_tokens_to_ids(__a),			[9, 6, 7, 12, 10, 11])







							def    snake_case__      (   self):

														'''simple docstring'''

														if not self.test_rust_tokenizer:
																					return

														_lowerCAmelCase     :							Optional[int]    =    self.get_tokenizer()
														_lowerCAmelCase     :							Union[str, Any]    =    self.get_rust_tokenizer()

														_lowerCAmelCase     :							Any    =    "UNwant\u00E9d,running"

														_lowerCAmelCase     :							Optional[Any]    =    tokenizer.tokenize(__a)
														_lowerCAmelCase     :							Optional[int]    =    rust_tokenizer.tokenize(__a)
														self.assertListEqual(__a,			__a)

														_lowerCAmelCase     :							List[Any]    =    tokenizer.encode(__a,			add_special_tokens=__a)
														_lowerCAmelCase     :							Optional[int]    =    rust_tokenizer.encode(__a,			add_special_tokens=__a)
														self.assertListEqual(__a,			__a)

														_lowerCAmelCase     :							List[str]    =    self.get_rust_tokenizer()
														_lowerCAmelCase     :							Dict    =    tokenizer.encode(__a)
														_lowerCAmelCase     :							str    =    rust_tokenizer.encode(__a)
														self.assertListEqual(__a,			__a)

														# With lower casing
														_lowerCAmelCase     :							Any    =    self.get_tokenizer(do_lower_case=__a)
														_lowerCAmelCase     :							Optional[int]    =    self.get_rust_tokenizer(do_lower_case=__a)

														_lowerCAmelCase     :							Union[str, Any]    =    "UNwant\u00E9d,running"

														_lowerCAmelCase     :							List[Any]    =    tokenizer.tokenize(__a)
														_lowerCAmelCase     :							Optional[Any]    =    rust_tokenizer.tokenize(__a)
														self.assertListEqual(__a,			__a)

														_lowerCAmelCase     :							Any    =    tokenizer.encode(__a,			add_special_tokens=__a)
														_lowerCAmelCase     :							Tuple    =    rust_tokenizer.encode(__a,			add_special_tokens=__a)
														self.assertListEqual(__a,			__a)

														_lowerCAmelCase     :							Optional[Any]    =    self.get_rust_tokenizer()
														_lowerCAmelCase     :							List[str]    =    tokenizer.encode(__a)
														_lowerCAmelCase     :							Optional[int]    =    rust_tokenizer.encode(__a)
														self.assertListEqual(__a,			__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    BasicTokenizer()

														self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"),			["ah", "\u535A", "\u63A8", "zz"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    BasicTokenizer(do_lower_case=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "),			["hello", "!", "how", "are", "you", "?"])
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"),			["hello"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Any    =    BasicTokenizer(do_lower_case=__a,			strip_accents=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "),			["hällo", "!", "how", "are", "you", "?"])
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"),			["h\u00E9llo"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							str    =    BasicTokenizer(do_lower_case=__a,			strip_accents=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "),			["hallo", "!", "how", "are", "you", "?"])
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"),			["hello"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    BasicTokenizer(do_lower_case=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "),			["hallo", "!", "how", "are", "you", "?"])
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"),			["hello"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    BasicTokenizer(do_lower_case=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "),			["HeLLo", "!", "how", "Are", "yoU", "?"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    BasicTokenizer(do_lower_case=__a,			strip_accents=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "),			["HäLLo", "!", "how", "Are", "yoU", "?"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							str    =    BasicTokenizer(do_lower_case=__a,			strip_accents=__a)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "),			["HaLLo", "!", "how", "Are", "yoU", "?"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							str    =    BasicTokenizer(do_lower_case=__a,			never_split=["[UNK]"])

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"),			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

														_lowerCAmelCase     :							int    =    {}
														for i, token in enumerate(__a):
																					_lowerCAmelCase     :							Dict    =    i
														_lowerCAmelCase     :							Dict    =    WordpieceTokenizer(vocab=__a,			unk_token="[UNK]")

														self.assertListEqual(tokenizer.tokenize(""),			[])

														self.assertListEqual(tokenizer.tokenize("unwanted running"),			["un", "##want", "##ed", "runn", "##ing"])

														self.assertListEqual(tokenizer.tokenize("unwantedX running"),			["[UNK]", "runn", "##ing"])







							def    snake_case__      (   self):

														'''simple docstring'''

														self.assertTrue(_is_whitespace(" "))
														self.assertTrue(_is_whitespace("\t"))
														self.assertTrue(_is_whitespace("\r"))
														self.assertTrue(_is_whitespace("\n"))
														self.assertTrue(_is_whitespace("\u00A0"))

														self.assertFalse(_is_whitespace("A"))
														self.assertFalse(_is_whitespace("-"))







							def    snake_case__      (   self):

														'''simple docstring'''

														self.assertTrue(_is_control("\u0005"))

														self.assertFalse(_is_control("A"))
														self.assertFalse(_is_control(" "))
														self.assertFalse(_is_control("\t"))
														self.assertFalse(_is_control("\r"))







							def    snake_case__      (   self):

														'''simple docstring'''

														self.assertTrue(_is_punctuation("-"))
														self.assertTrue(_is_punctuation("$"))
														self.assertTrue(_is_punctuation("`"))
														self.assertTrue(_is_punctuation("."))

														self.assertFalse(_is_punctuation("A"))
														self.assertFalse(_is_punctuation(" "))







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    self.get_tokenizer()
														_lowerCAmelCase     :							int    =    self.get_rust_tokenizer()

														# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
														self.assertListEqual([tokenizer.tokenize(__a) for t in ["Test", "\xad", "test"]],			[["[UNK]"], [], ["[UNK]"]])

														self.assertListEqual(
														    [rust_tokenizer.tokenize(__a) for t in ["Test", "\xad", "test"]],			[["[UNK]"], [], ["[UNK]"]])







							@slow
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    self.tokenizer_class.from_pretrained("google/mobilebert-uncased")

														_lowerCAmelCase     :							Tuple    =    tokenizer.encode("sequence builders",			add_special_tokens=__a)
														_lowerCAmelCase     :							Any    =    tokenizer.encode("multi-sequence build",			add_special_tokens=__a)

														_lowerCAmelCase     :							Dict    =    tokenizer.build_inputs_with_special_tokens(__a)
														_lowerCAmelCase     :							int    =    tokenizer.build_inputs_with_special_tokens(__a,			__a)

														assert encoded_sentence == [101] + text + [102]
														assert encoded_pair == [101] + text + [102] + text_a + [102]







							def    snake_case__      (   self):

														'''simple docstring'''

														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
																												_lowerCAmelCase     :							Union[str, Any]    =    self.rust_tokenizer_class.from_pretrained(__a,			**__a)

																												_lowerCAmelCase     :							Optional[int]    =    f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																												_lowerCAmelCase     :							List[Any]    =    tokenizer_r.encode_plus(
																												    __a,			return_attention_mask=__a,			return_token_type_ids=__a,			return_offsets_mapping=__a,			add_special_tokens=__a,			)

																												_lowerCAmelCase     :							Tuple    =    tokenizer_r.do_lower_case if hasattr(__a,			"do_lower_case") else False
																												_lowerCAmelCase     :							Tuple    =    (
																												    [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), "A"),
																												        ((1, 2), ","),
																												        ((3, 5), "na"),
																												        ((5, 6), "##ï"),
																												        ((6, 8), "##ve"),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), "Allen"),
																												        ((21, 23), "##NL"),
																												        ((23, 24), "##P"),
																												        ((25, 33), "sentence"),
																												        ((33, 34), "."),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												    if not do_lower_case
																												    else [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), "a"),
																												        ((1, 2), ","),
																												        ((3, 8), "naive"),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), "allen"),
																												        ((21, 23), "##nl"),
																												        ((23, 24), "##p"),
																												        ((25, 33), "sentence"),
																												        ((33, 34), "."),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												)

																												self.assertEqual(
																												    [e[1] for e in expected_results],			tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
																												self.assertEqual([e[0] for e in expected_results],			tokens["offset_mapping"])







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    ["的", "人", "有"]
														_lowerCAmelCase     :							Tuple    =    "".join(__a)
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
																												_lowerCAmelCase     :							Dict    =    True
																												_lowerCAmelCase     :							Optional[int]    =    self.tokenizer_class.from_pretrained(__a,			**__a)
																												_lowerCAmelCase     :							str    =    self.rust_tokenizer_class.from_pretrained(__a,			**__a)

																												_lowerCAmelCase     :							Any    =    tokenizer_p.encode(__a,			add_special_tokens=__a)
																												_lowerCAmelCase     :							Union[str, Any]    =    tokenizer_r.encode(__a,			add_special_tokens=__a)

																												_lowerCAmelCase     :							Optional[int]    =    tokenizer_r.convert_ids_to_tokens(__a)
																												_lowerCAmelCase     :							Union[str, Any]    =    tokenizer_p.convert_ids_to_tokens(__a)

																												# it is expected that each Chinese character is not preceded by "##"
																												self.assertListEqual(__a,			__a)
																												self.assertListEqual(__a,			__a)

																												_lowerCAmelCase     :							Optional[Any]    =    False
																												_lowerCAmelCase     :							Tuple    =    self.rust_tokenizer_class.from_pretrained(__a,			**__a)
																												_lowerCAmelCase     :							int    =    self.tokenizer_class.from_pretrained(__a,			**__a)

																												_lowerCAmelCase     :							Optional[Any]    =    tokenizer_r.encode(__a,			add_special_tokens=__a)
																												_lowerCAmelCase     :							str    =    tokenizer_p.encode(__a,			add_special_tokens=__a)

																												_lowerCAmelCase     :							List[str]    =    tokenizer_r.convert_ids_to_tokens(__a)
																												_lowerCAmelCase     :							Any    =    tokenizer_p.convert_ids_to_tokens(__a)

																												# it is expected that only the first Chinese character is not preceded by "##".
																												_lowerCAmelCase     :							Union[str, Any]    =    [
																												    f"##{token}" if idx != 0 else token for idx, token in enumerate(__a)
																												]
																												self.assertListEqual(__a,			__a)
																												self.assertListEqual(__a,			__a)





import argparse
from collections import defaultdict

import yaml


_snake_case		=  "docs/source/en/_toctree.yml"






def 				A				(					_lowerCamelCase  ):





							'''simple docstring'''




							_lowerCAmelCase     :							Dict    =    defaultdict(_lowerCamelCase  )
							_lowerCAmelCase     :							Any    =    []
							_lowerCAmelCase     :							List[str]    =    []
							for doc in doc_list:
														if "local" in doc:
																					counts[doc["local"]] += 1

														if doc["title"].lower() == "overview":
																					overview_doc.append({"local": doc["local"], "title": doc["title"]}  )
														else:
																					new_doc_list.append(_lowerCamelCase  )

							_lowerCAmelCase     :							Optional[Any]    =    new_doc_list
							_lowerCAmelCase     :							List[Any]    =    [key for key, value in counts.items() if value > 1]

							_lowerCAmelCase     :							str    =    []
							for duplicate_key in duplicates:
														_lowerCAmelCase     :							List[str]    =    list({doc["title"] for doc in doc_list if doc["local"] == duplicate_key}  )
														if len(_lowerCamelCase  ) > 1:
																					raise ValueError(
																					    F"{duplicate_key} is present several times in the documentation table of content at "
																					    "`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the "
																					    "others."  )
														# Only add this once
														new_doc.append({"local": duplicate_key, "title": titles[0]}  )

							# Add none duplicate-keys
							new_doc.extend([doc for doc in doc_list if "local" not in counts or counts[doc["local"]] == 1]  )
							_lowerCAmelCase     :							Optional[Any]    =    sorted(_lowerCamelCase				, key=lambda _lowerCamelCase  : s["title"].lower()  )

							# "overview" gets special treatment and is always first
							if len(_lowerCamelCase  ) > 1:
														raise ValueError("{doc_list} has two 'overview' docs which is not allowed."  )

							overview_doc.extend(_lowerCamelCase  )

							# Sort
							return overview_doc






def 				A				(					_lowerCamelCase=False  ):





							'''simple docstring'''




							with open(_lowerCamelCase				, encoding="utf-8"  ) as f:
														_lowerCAmelCase     :							int    =    yaml.safe_load(f.read()  )

							# Get to the API doc
							_lowerCAmelCase     :							Optional[Any]    =    0
							while content[api_idx]["title"] != "API":
														api_idx += 1
							_lowerCAmelCase     :							List[str]    =    content[api_idx]["sections"]

							# Then to the model doc
							_lowerCAmelCase     :							Union[str, Any]    =    0
							while api_doc[scheduler_idx]["title"] != "Schedulers":
														scheduler_idx += 1

							_lowerCAmelCase     :							Optional[Any]    =    api_doc[scheduler_idx]["sections"]
							_lowerCAmelCase     :							Optional[Any]    =    clean_doc_toc(_lowerCamelCase  )

							_lowerCAmelCase     :							int    =    False
							if new_scheduler_doc != scheduler_doc:
														_lowerCAmelCase     :							List[Any]    =    True
														if overwrite:
																					_lowerCAmelCase     :							Dict    =    new_scheduler_doc

							if diff:
														if overwrite:
																					_lowerCAmelCase     :							Tuple    =    api_doc
																					with open(_lowerCamelCase				, "w"				, encoding="utf-8"  ) as f:
																												f.write(yaml.dump(_lowerCamelCase				, allow_unicode=_lowerCamelCase  )  )
														else:
																					raise ValueError(
																					    "The model doc part of the table of content is not properly sorted, run `make style` to fix this."  )






def 				A				(					_lowerCamelCase=False  ):





							'''simple docstring'''




							with open(_lowerCamelCase				, encoding="utf-8"  ) as f:
														_lowerCAmelCase     :							Tuple    =    yaml.safe_load(f.read()  )

							# Get to the API doc
							_lowerCAmelCase     :							Optional[int]    =    0
							while content[api_idx]["title"] != "API":
														api_idx += 1
							_lowerCAmelCase     :							int    =    content[api_idx]["sections"]

							# Then to the model doc
							_lowerCAmelCase     :							List[str]    =    0
							while api_doc[pipeline_idx]["title"] != "Pipelines":
														pipeline_idx += 1

							_lowerCAmelCase     :							Dict    =    False
							_lowerCAmelCase     :							Optional[int]    =    api_doc[pipeline_idx]["sections"]
							_lowerCAmelCase     :							Tuple    =    []

							# sort sub pipeline docs
							for pipeline_doc in pipeline_docs:
														if "section" in pipeline_doc:
																					_lowerCAmelCase     :							List[Any]    =    pipeline_doc["section"]
																					_lowerCAmelCase     :							Union[str, Any]    =    clean_doc_toc(_lowerCamelCase  )
																					if overwrite:
																												_lowerCAmelCase     :							Optional[Any]    =    new_sub_pipeline_doc
														new_pipeline_docs.append(_lowerCamelCase  )

							# sort overall pipeline doc
							_lowerCAmelCase     :							Union[str, Any]    =    clean_doc_toc(_lowerCamelCase  )

							if new_pipeline_docs != pipeline_docs:
														_lowerCAmelCase     :							Dict    =    True
														if overwrite:
																					_lowerCAmelCase     :							Optional[int]    =    new_pipeline_docs

							if diff:
														if overwrite:
																					_lowerCAmelCase     :							Optional[int]    =    api_doc
																					with open(_lowerCamelCase				, "w"				, encoding="utf-8"  ) as f:
																												f.write(yaml.dump(_lowerCamelCase				, allow_unicode=_lowerCamelCase  )  )
														else:
																					raise ValueError(
																					    "The model doc part of the table of content is not properly sorted, run `make style` to fix this."  )


if __name__ == "__main__":
						_snake_case		=  argparse.ArgumentParser()
						parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
						_snake_case		=  parser.parse_args()

						check_scheduler_doc(args.fix_and_overwrite)
						check_pipeline_doc(args.fix_and_overwrite)
