



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase      :							Optional[Any]							       =get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class 					_lowercase      (a_					,      unittest.TestCase					):
				'''simple docstring'''







				lowercase__        =  XLMRobertaTokenizer
				lowercase__        =  XLMRobertaTokenizerFast
				lowercase__        =  True
				lowercase__        =  True

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									super().setUp()

									# We have a SentencePiece fixture for testing
									UpperCamelCase_						     =      XLMRobertaTokenizer(__a   ,       keep_accents=__a			)
									tokenizer.save_pretrained(self.tmpdirname			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      "<pad>"
									UpperCamelCase_						     =      1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a			)   ,       __a			)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a			)   ,       __a			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      list(self.get_tokenizer().get_vocab().keys()			)

									self.assertEqual(vocab_keys[0]   ,       "<s>"			)
									self.assertEqual(vocab_keys[1]   ,       "<pad>"			)
									self.assertEqual(vocab_keys[-1]   ,       "<mask>"			)
									self.assertEqual(len(__a			)   ,       1002			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									self.assertEqual(self.get_tokenizer().vocab_size   ,       1002			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      XLMRobertaTokenizer(__a   ,       keep_accents=__a			)

									UpperCamelCase_						     =      tokenizer.tokenize("This is a test"			)
									self.assertListEqual(__a   ,       ["▁This", "▁is", "▁a", "▁t", "est"]			)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(__a			)   ,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,       )

									UpperCamelCase_						     =      tokenizer.tokenize("I was born in 92000, and this is falsé."			)
									self.assertListEqual(
									    __a   ,       [
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "é",
									        ".",
									    ]   ,       )
									UpperCamelCase_						     =      tokenizer.convert_tokens_to_ids(__a			)
									self.assertListEqual(
									    __a   ,       [
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
									    ]   ,       )

									UpperCamelCase_						     =      tokenizer.convert_ids_to_tokens(__a			)
									self.assertListEqual(
									    __a   ,       [
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "<unk>",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "<unk>",
									        ".",
									    ]   ,       )

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									if not self.test_slow_tokenizer:
														# as we don't have a slow version, we can't compare the outputs between slow and fast versions
														return

									UpperCamelCase_						     =      (self.rust_tokenizer_class, "hf-internal-testing/tiny-xlm-roberta", {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																			UpperCamelCase_						     =      self.rust_tokenizer_class.from_pretrained(__a   ,       **__a			)
																			UpperCamelCase_						     =      self.tokenizer_class.from_pretrained(__a   ,       **__a			)

																			UpperCamelCase_						     =      tempfile.mkdtemp()

																			UpperCamelCase_						     =      tokenizer_r.save_pretrained(__a			)
																			UpperCamelCase_						     =      tokenizer_p.save_pretrained(__a			)

																			# Checks it save with the same files + the tokenizer.json file for the fast one
																			self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files			)			)
																			UpperCamelCase_						     =      tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f			)
																			self.assertSequenceEqual(__a   ,       __a			)

																			# Checks everything loads correctly in the same way
																			UpperCamelCase_						     =      tokenizer_r.from_pretrained(__a			)
																			UpperCamelCase_						     =      tokenizer_p.from_pretrained(__a			)

																			# Check special tokens are set accordingly on Rust and Python
																			for key in tokenizer_pp.special_tokens_map:
																								self.assertTrue(hasattr(__a   ,       __a			)			)
																								# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																								# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																			shutil.rmtree(__a			)

																			# Save tokenizer rust, legacy_format=True
																			UpperCamelCase_						     =      tempfile.mkdtemp()

																			UpperCamelCase_						     =      tokenizer_r.save_pretrained(__a   ,       legacy_format=__a			)
																			UpperCamelCase_						     =      tokenizer_p.save_pretrained(__a			)

																			# Checks it save with the same files
																			self.assertSequenceEqual(__a   ,       __a			)

																			# Checks everything loads correctly in the same way
																			UpperCamelCase_						     =      tokenizer_r.from_pretrained(__a			)
																			UpperCamelCase_						     =      tokenizer_p.from_pretrained(__a			)

																			# Check special tokens are set accordingly on Rust and Python
																			for key in tokenizer_pp.special_tokens_map:
																								self.assertTrue(hasattr(__a   ,       __a			)			)

																			shutil.rmtree(__a			)

																			# Save tokenizer rust, legacy_format=False
																			UpperCamelCase_						     =      tempfile.mkdtemp()

																			UpperCamelCase_						     =      tokenizer_r.save_pretrained(__a   ,       legacy_format=__a			)
																			UpperCamelCase_						     =      tokenizer_p.save_pretrained(__a			)

																			# Checks it saved the tokenizer.json file
																			self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files			)			)

																			# Checks everything loads correctly in the same way
																			UpperCamelCase_						     =      tokenizer_r.from_pretrained(__a			)
																			UpperCamelCase_						     =      tokenizer_p.from_pretrained(__a			)

																			# Check special tokens are set accordingly on Rust and Python
																			for key in tokenizer_pp.special_tokens_map:
																								self.assertTrue(hasattr(__a   ,       __a			)			)

																			shutil.rmtree(__a			)

				@cached_property
				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									return XLMRobertaTokenizer.from_pretrained("xlm-roberta-base"			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									with tempfile.NamedTemporaryFile() as f:
														shutil.copyfile(__a   ,       f.name			)
														UpperCamelCase_						     =      XLMRobertaTokenizer(f.name   ,       keep_accents=__a			)
														UpperCamelCase_						     =      pickle.dumps(__a			)
									pickle.loads(__a			)

				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									if not self.test_rust_tokenizer:
														return

									UpperCamelCase_						     =      self.get_tokenizer()
									UpperCamelCase_						     =      self.get_rust_tokenizer()

									UpperCamelCase_						     =      "I was born in 92000, and this is falsé."

									UpperCamelCase_						     =      tokenizer.tokenize(__a			)
									UpperCamelCase_						     =      rust_tokenizer.tokenize(__a			)
									self.assertListEqual(__a   ,       __a			)

									UpperCamelCase_						     =      tokenizer.encode(__a   ,       add_special_tokens=__a			)
									UpperCamelCase_						     =      rust_tokenizer.encode(__a   ,       add_special_tokens=__a			)
									self.assertListEqual(__a   ,       __a			)

									UpperCamelCase_						     =      self.get_rust_tokenizer()
									UpperCamelCase_						     =      tokenizer.encode(__a			)
									UpperCamelCase_						     =      rust_tokenizer.encode(__a			)
									self.assertListEqual(__a   ,       __a			)

				@slow
				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      "Hello World!"
									UpperCamelCase_						     =      [0, 3_5378, 6661, 38, 2]
									# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
									# xlmr.eval()
									# xlmr.encode(symbols)

									self.assertListEqual(__a   ,       self.big_tokenizer.encode(__a			)			)

				@slow
				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      (
									    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
									    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
									)
									UpperCamelCase_						     =      [
									    0,
									    3293,
									    83,
									    10,
									    4552,
									    4989,
									    7986,
									    678,
									    10,
									    5915,
									    111,
									    17_9459,
									    12_4850,
									    4,
									    6044,
									    237,
									    12,
									    6,
									    5,
									    6,
									    4,
									    6780,
									    705,
									    15,
									    1388,
									    44,
									    378,
									    1_0114,
									    711,
									    152,
									    20,
									    6,
									    5,
									    2_2376,
									    642,
									    1221,
									    1_5190,
									    3_4153,
									    450,
									    5608,
									    959,
									    1119,
									    5_7702,
									    136,
									    186,
									    47,
									    1098,
									    2_9367,
									    47,
									    # 4426, # What fairseq tokenizes from "<unk>": "_<"
									    # 3678, # What fairseq tokenizes from "<unk>": "unk"
									    # 2740, # What fairseq tokenizes from "<unk>": ">"
									    3,  # What we tokenize from "<unk>": "<unk>"
									    6,  # Residue from the tokenization: an extra sentencepiece underline
									    4,
									    6044,
									    237,
									    6284,
									    5_0901,
									    528,
									    31,
									    90,
									    34,
									    927,
									    2,
									]
									# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
									# xlmr.eval()
									# xlmr.encode(symbols)

									self.assertListEqual(__a   ,       self.big_tokenizer.encode(__a			)			)

				@slow
				def 					_lowerCamelCase    (       self			):

									'''simple docstring'''


									UpperCamelCase_						     =      {"input_ids": [[0, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [0, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=__a   ,       model_name="xlm-roberta-base"   ,       revision="d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"   ,       )





import argparse
from collections import defaultdict

import yaml


_snake_case		=  "docs/source/en/_toctree.yml"






def 				A				(					_lowerCamelCase  ):





							'''simple docstring'''




							_lowerCAmelCase     :							Dict    =    defaultdict(_lowerCamelCase  )
							_lowerCAmelCase     :							Any    =    []
							_lowerCAmelCase     :							List[str]    =    []
							for doc in doc_list:
														if "local" in doc:
																					counts[doc["local"]] += 1

														if doc["title"].lower() == "overview":
																					overview_doc.append({"local": doc["local"], "title": doc["title"]}  )
														else:
																					new_doc_list.append(_lowerCamelCase  )

							_lowerCAmelCase     :							Optional[Any]    =    new_doc_list
							_lowerCAmelCase     :							List[Any]    =    [key for key, value in counts.items() if value > 1]

							_lowerCAmelCase     :							str    =    []
							for duplicate_key in duplicates:
														_lowerCAmelCase     :							List[str]    =    list({doc["title"] for doc in doc_list if doc["local"] == duplicate_key}  )
														if len(_lowerCamelCase  ) > 1:
																					raise ValueError(
																					    F"{duplicate_key} is present several times in the documentation table of content at "
																					    "`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the "
																					    "others."  )
														# Only add this once
														new_doc.append({"local": duplicate_key, "title": titles[0]}  )

							# Add none duplicate-keys
							new_doc.extend([doc for doc in doc_list if "local" not in counts or counts[doc["local"]] == 1]  )
							_lowerCAmelCase     :							Optional[Any]    =    sorted(_lowerCamelCase				, key=lambda _lowerCamelCase  : s["title"].lower()  )

							# "overview" gets special treatment and is always first
							if len(_lowerCamelCase  ) > 1:
														raise ValueError("{doc_list} has two 'overview' docs which is not allowed."  )

							overview_doc.extend(_lowerCamelCase  )

							# Sort
							return overview_doc






def 				A				(					_lowerCamelCase=False  ):





							'''simple docstring'''




							with open(_lowerCamelCase				, encoding="utf-8"  ) as f:
														_lowerCAmelCase     :							int    =    yaml.safe_load(f.read()  )

							# Get to the API doc
							_lowerCAmelCase     :							Optional[Any]    =    0
							while content[api_idx]["title"] != "API":
														api_idx += 1
							_lowerCAmelCase     :							List[str]    =    content[api_idx]["sections"]

							# Then to the model doc
							_lowerCAmelCase     :							Union[str, Any]    =    0
							while api_doc[scheduler_idx]["title"] != "Schedulers":
														scheduler_idx += 1

							_lowerCAmelCase     :							Optional[Any]    =    api_doc[scheduler_idx]["sections"]
							_lowerCAmelCase     :							Optional[Any]    =    clean_doc_toc(_lowerCamelCase  )

							_lowerCAmelCase     :							int    =    False
							if new_scheduler_doc != scheduler_doc:
														_lowerCAmelCase     :							List[Any]    =    True
														if overwrite:
																					_lowerCAmelCase     :							Dict    =    new_scheduler_doc

							if diff:
														if overwrite:
																					_lowerCAmelCase     :							Tuple    =    api_doc
																					with open(_lowerCamelCase				, "w"				, encoding="utf-8"  ) as f:
																												f.write(yaml.dump(_lowerCamelCase				, allow_unicode=_lowerCamelCase  )  )
														else:
																					raise ValueError(
																					    "The model doc part of the table of content is not properly sorted, run `make style` to fix this."  )






def 				A				(					_lowerCamelCase=False  ):





							'''simple docstring'''




							with open(_lowerCamelCase				, encoding="utf-8"  ) as f:
														_lowerCAmelCase     :							Tuple    =    yaml.safe_load(f.read()  )

							# Get to the API doc
							_lowerCAmelCase     :							Optional[int]    =    0
							while content[api_idx]["title"] != "API":
														api_idx += 1
							_lowerCAmelCase     :							int    =    content[api_idx]["sections"]

							# Then to the model doc
							_lowerCAmelCase     :							List[str]    =    0
							while api_doc[pipeline_idx]["title"] != "Pipelines":
														pipeline_idx += 1

							_lowerCAmelCase     :							Dict    =    False
							_lowerCAmelCase     :							Optional[int]    =    api_doc[pipeline_idx]["sections"]
							_lowerCAmelCase     :							Tuple    =    []

							# sort sub pipeline docs
							for pipeline_doc in pipeline_docs:
														if "section" in pipeline_doc:
																					_lowerCAmelCase     :							List[Any]    =    pipeline_doc["section"]
																					_lowerCAmelCase     :							Union[str, Any]    =    clean_doc_toc(_lowerCamelCase  )
																					if overwrite:
																												_lowerCAmelCase     :							Optional[Any]    =    new_sub_pipeline_doc
														new_pipeline_docs.append(_lowerCamelCase  )

							# sort overall pipeline doc
							_lowerCAmelCase     :							Union[str, Any]    =    clean_doc_toc(_lowerCamelCase  )

							if new_pipeline_docs != pipeline_docs:
														_lowerCAmelCase     :							Dict    =    True
														if overwrite:
																					_lowerCAmelCase     :							Optional[int]    =    new_pipeline_docs

							if diff:
														if overwrite:
																					_lowerCAmelCase     :							Optional[int]    =    api_doc
																					with open(_lowerCamelCase				, "w"				, encoding="utf-8"  ) as f:
																												f.write(yaml.dump(_lowerCamelCase				, allow_unicode=_lowerCamelCase  )  )
														else:
																					raise ValueError(
																					    "The model doc part of the table of content is not properly sorted, run `make style` to fix this."  )


if __name__ == "__main__":
						_snake_case		=  argparse.ArgumentParser()
						parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
						_snake_case		=  parser.parse_args()

						check_scheduler_doc(args.fix_and_overwrite)
						check_pipeline_doc(args.fix_and_overwrite)
