'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast

class 	lowerCamelCase_	(unittest.TestCase    ):
      '''simple docstring'''

      def      _A	(       self						:				List[str]       ):
             # A mock response for an HTTP head request to emulate server down
             _UpperCAmelCase     :  Optional[Any]							      =    mock.Mock()
             _UpperCAmelCase     :  Optional[int]							      =    500
             _UpperCAmelCase     :  Tuple							      =    {}
             _UpperCAmelCase     :  Optional[int]							      =    HTTPError
             _UpperCAmelCase     :  Any							      =    {}

             # Download this model to make sure it's in the cache.
             _UpperCAmelCase     :  List[str]							      =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )

             # Under the mock environment we get a 500 error when trying to reach the tokenizer.
             with mock.patch("requests.Session.request"						,					return_value=A       ) as mock_head:
                    _UpperCAmelCase     :  Union[str, Any]							      =    BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"       )
                    # This check we did call the fake head request
                    mock_head.assert_called()




      @require_tokenizers
      def      _A	(       self						:				int       ):
             # A mock response for an HTTP head request to emulate server down
             _UpperCAmelCase     :  List[Any]							      =    mock.Mock()
             _UpperCAmelCase     :  Union[str, Any]							      =    500
             _UpperCAmelCase     :  Tuple							      =    {}
             _UpperCAmelCase     :  Union[str, Any]							      =    HTTPError
             _UpperCAmelCase     :  Optional[Any]							      =    {}

             # Download this model to make sure it's in the cache.
             _UpperCAmelCase     :  Any							      =    GPTaTokenizerFast.from_pretrained("gpt2"       )

             # Under the mock environment we get a 500 error when trying to reach the tokenizer.
             with mock.patch("requests.Session.request"						,					return_value=A       ) as mock_head:
                    _UpperCAmelCase     :  Dict							      =    GPTaTokenizerFast.from_pretrained("gpt2"       )
                    # This check we did call the fake head request
                    mock_head.assert_called()




      def      _A	(       self						:				str       ):
             # This test is for deprecated behavior and can be removed in v5
             try:
                    _UpperCAmelCase     :  Any							      =    tempfile.mktemp()
                    with open(A						,					"wb"       ) as f:
                           http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"						,					A       )

                    _UpperCAmelCase     :  Union[str, Any]							      =    AlbertTokenizer.from_pretrained(A       )
             finally:
                    os.remove(A       )

             # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
             # the current folder and have the right name.
             if os.path.isfile("tokenizer.json"       ):
                    # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                    return
             try:
                    with open("tokenizer.json"						,					"wb"       ) as f:
                           http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"						,					A       )
                    _UpperCAmelCase     :  Optional[Any]							      =    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"       )
                    # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                    self.assertEqual(tokenizer.vocab_size						,					1000       )
                    # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

             finally:
                    os.remove("tokenizer.json"       )





      def      _A	(       self						:				Any       ):
             # This test is for deprecated behavior and can be removed in v5
             _UpperCAmelCase     :  Optional[Any]							      =    AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       )

@is_staging_test
class 	lowerCamelCase_	(unittest.TestCase    ):
      '''simple docstring'''

      __UpperCamelCase:  Dict						=						["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




      @classmethod
      def      _A	(       cls						:				List[Any]       ):
             _UpperCAmelCase     :  Union[str, Any]							      =    TOKEN
             HfFolder.save_token(A       )




      @classmethod
      def      _A	(       cls						:				int       ):
             try:
                    delete_repo(token=cls._token						,					repo_id="test-tokenizer"       )
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token						,					repo_id="valid_org/test-tokenizer-org"       )
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token						,					repo_id="test-dynamic-tokenizer"       )
             except HTTPError:
                    pass




      def      _A	(       self						:				Optional[int]       ):
             with tempfile.TemporaryDirectory() as tmp_dir:
                    _UpperCAmelCase     :  Any							      =    os.path.join(A						,					"vocab.txt"       )
                    with open(A						,					"w"						,					encoding="utf-8"       ) as vocab_writer:
                           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
                    _UpperCAmelCase     :  Optional[Any]							      =    BertTokenizer(A       )

             tokenizer.push_to_hub("test-tokenizer"						,					use_auth_token=self._token       )
             _UpperCAmelCase     :  Optional[Any]							      =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""       )
             self.assertDictEqual(new_tokenizer.vocab						,					tokenizer.vocab       )

             # Reset repo
             delete_repo(token=self._token						,					repo_id="test-tokenizer"       )

             # Push to hub via save_pretrained
             with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(A						,					repo_id="test-tokenizer"						,					push_to_hub=A						,					use_auth_token=self._token       )

             _UpperCAmelCase     :  Any							      =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""       )
             self.assertDictEqual(new_tokenizer.vocab						,					tokenizer.vocab       )




      def      _A	(       self						:				List[Any]       ):
             with tempfile.TemporaryDirectory() as tmp_dir:
                    _UpperCAmelCase     :  Optional[Any]							      =    os.path.join(A						,					"vocab.txt"       )
                    with open(A						,					"w"						,					encoding="utf-8"       ) as vocab_writer:
                           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
                    _UpperCAmelCase     :  str							      =    BertTokenizer(A       )

             tokenizer.push_to_hub("valid_org/test-tokenizer-org"						,					use_auth_token=self._token       )
             _UpperCAmelCase     :  int							      =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
             self.assertDictEqual(new_tokenizer.vocab						,					tokenizer.vocab       )

             # Reset repo
             delete_repo(token=self._token						,					repo_id="valid_org/test-tokenizer-org"       )

             # Push to hub via save_pretrained
             with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(
                        A						,					repo_id="valid_org/test-tokenizer-org"						,					push_to_hub=A						,					use_auth_token=self._token       )

             _UpperCAmelCase     :  Optional[Any]							      =    BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"       )
             self.assertDictEqual(new_tokenizer.vocab						,					tokenizer.vocab       )





      @require_tokenizers
      def      _A	(       self						:				List[Any]       ):
             CustomTokenizer.register_for_auto_class()
             with tempfile.TemporaryDirectory() as tmp_dir:
                    _UpperCAmelCase     :  Tuple							      =    os.path.join(A						,					"vocab.txt"       )
                    with open(A						,					"w"						,					encoding="utf-8"       ) as vocab_writer:
                           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )
                    _UpperCAmelCase     :  List[Any]							      =    CustomTokenizer(A       )

             # No fast custom tokenizer
             tokenizer.push_to_hub("test-dynamic-tokenizer"						,					use_auth_token=self._token       )

             _UpperCAmelCase     :  Optional[int]							      =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,					trust_remote_code=A       )
             # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
             self.assertEqual(tokenizer.__class__.__name__						,					"CustomTokenizer"       )

             # Fast and slow custom tokenizer
             CustomTokenizerFast.register_for_auto_class()
             with tempfile.TemporaryDirectory() as tmp_dir:
                    _UpperCAmelCase     :  Union[str, Any]							      =    os.path.join(A						,					"vocab.txt"       )
                    with open(A						,					"w"						,					encoding="utf-8"       ) as vocab_writer:
                           vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]       )       )

                    _UpperCAmelCase     :  Optional[Any]							      =    BertTokenizerFast.from_pretrained(A       )
                    bert_tokenizer.save_pretrained(A       )
                    _UpperCAmelCase     :  List[str]							      =    CustomTokenizerFast.from_pretrained(A       )

             tokenizer.push_to_hub("test-dynamic-tokenizer"						,					use_auth_token=self._token       )

             _UpperCAmelCase     :  Union[str, Any]							      =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""						,					trust_remote_code=A       )
             # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
             self.assertEqual(tokenizer.__class__.__name__						,					"CustomTokenizerFast"       )
             _UpperCAmelCase     :  Tuple							      =    AutoTokenizer.from_pretrained(
                 F"""{USER}/test-dynamic-tokenizer"""						,					use_fast=A						,					trust_remote_code=A       )
             # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
             self.assertEqual(tokenizer.__class__.__name__						,					"CustomTokenizer"       )

class 	lowerCamelCase_	(unittest.TestCase    ):
      '''simple docstring'''

      def      _A	(       self						:				Optional[Any]       ):
             _UpperCAmelCase     :  Optional[Any]							      =    Trie()
             trie.add("Hello 友達"       )
             self.assertEqual(trie.data						,					{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}       )
             trie.add("Hello"       )
             trie.data
             self.assertEqual(trie.data						,					{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}       )




      def      _A	(       self						:				List[Any]       ):
             _UpperCAmelCase     :  Union[str, Any]							      =    Trie()
             self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )						,					["[CLS] This is a extra_id_100"]       )
             trie.add("[CLS]"       )
             trie.add("extra_id_1"       )
             trie.add("extra_id_100"       )
             self.assertEqual(trie.split("[CLS] This is a extra_id_100"       )						,					["[CLS]", " This is a ", "extra_id_100"]       )




      def      _A	(       self						:				Union[str, Any]       ):
             _UpperCAmelCase     :  int							      =    Trie()
             trie.add("A"       )
             self.assertEqual(trie.split("ABC"       )						,					["A", "BC"]       )
             self.assertEqual(trie.split("BCA"       )						,					["BC", "A"]       )




      def      _A	(       self						:				Union[str, Any]       ):
             _UpperCAmelCase     :  Tuple							      =    Trie()
             trie.add("TOKEN]"       )
             trie.add("[SPECIAL_TOKEN]"       )
             self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )						,					["This is something ", "[SPECIAL_TOKEN]"]       )




      def      _A	(       self						:				Optional[Any]       ):
             _UpperCAmelCase     :  int							      =    Trie()
             trie.add("A"       )
             trie.add("P"       )
             trie.add("[SPECIAL_TOKEN]"       )
             self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"       )						,					["This is something ", "[SPECIAL_TOKEN]"]       )




      def      _A	(       self						:				Optional[int]       ):
             _UpperCAmelCase     :  int							      =    Trie()
             trie.add("AB"       )
             trie.add("B"       )
             trie.add("C"       )
             self.assertEqual(trie.split("ABC"       )						,					["AB", "C"]       )




      def      _A	(       self						:				List[Any]       ):
             _UpperCAmelCase     :  List[Any]							      =    Trie()
             trie.add("ABC"       )
             trie.add("B"       )
             trie.add("CD"       )
             self.assertEqual(trie.split("ABCD"       )						,					["ABC", "D"]       )





      def      _A	(       self						:				Optional[Any]       ):
             # Even if the offsets are wrong, we necessarily output correct string
             # parts.
             _UpperCAmelCase     :  List[str]							      =    Trie()
             _UpperCAmelCase     :  Optional[int]							      =    trie.cut_text("ABC"						,					[0, 0, 2, 1, 2, 3]       )
             self.assertEqual(A						,					["AB", "C"]       )


'''simple docstring'''




import numpy as np
from matplotlib import pyplot as plt
from sklearn.datasets import load_iris
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier

def 	UpperCamelCase_							(					_UpperCAmelCase  :      dict    )    ->    tuple:


       """simple docstring"""







       return (data["data"], data["target"])

def 	UpperCamelCase_							(					_UpperCAmelCase  :      np.ndarray			,	_UpperCAmelCase  :      np.ndarray    )    ->    XGBClassifier:


       """simple docstring"""







       _UpperCAmelCase     :  Any							      =    XGBClassifier()
       classifier.fit(_UpperCAmelCase			,	_UpperCAmelCase    )
       return classifier

def 	UpperCamelCase_							(					)    ->    None:


       """simple docstring"""







       _UpperCAmelCase     :  List[str]							      =    load_iris()
       _UpperCAmelCase      ,  _UpperCAmelCase     :  Dict							      =    data_handling(_UpperCAmelCase    )
       _UpperCAmelCase      ,  _UpperCAmelCase      ,  _UpperCAmelCase      ,  _UpperCAmelCase     :  str							      =    train_test_split(
           _UpperCAmelCase			,	_UpperCAmelCase			,	test_size=0.2_5    )

       _UpperCAmelCase     :  Optional[Any]							      =    iris["target_names"]

       # Create an XGBoost Classifier from the training data
       _UpperCAmelCase     :  Tuple							      =    xgboost(_UpperCAmelCase			,	_UpperCAmelCase    )

       # Display the confusion matrix of the classifier with both training and test sets
       ConfusionMatrixDisplay.from_estimator(
           _UpperCAmelCase			,	_UpperCAmelCase			,	_UpperCAmelCase			,	display_labels=_UpperCAmelCase			,	cmap="Blues"			,	normalize="true"			,	)
       plt.title("Normalized Confusion Matrix - IRIS Dataset"    )
       plt.show()


if __name__ == "__main__":
  import doctest

  doctest.testmod(verbose=True)
  main()

