




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast
class 						_A					(  unittest.TestCase      ):






  def 					__a			(  self		:	List[str]      )		->    Tuple:






       """simple docstring"""







       lowercase					:   Optional[Any]								= mock.Mock()
       lowercase					:   Optional[int]								= 500
       lowercase					:   Tuple								= {}
       lowercase					:   Optional[int]								= HTTPError
       lowercase					:   Any								= {}

       # Download this model to make sure it's in the cache.
       lowercase					:   List[str]								= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request'''      ,						return_value=_A      ) as mock_head:
            lowercase					:   Union[str, Any]								= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
            # This check we did call the fake head request
            mock_head.assert_called()






  @require_tokenizers
  def 					__a			(  self		:	int      )		->    int:






       """simple docstring"""







       lowercase					:   List[Any]								= mock.Mock()
       lowercase					:   Union[str, Any]								= 500
       lowercase					:   Tuple								= {}
       lowercase					:   Union[str, Any]								= HTTPError
       lowercase					:   Optional[Any]								= {}

       # Download this model to make sure it's in the cache.
       lowercase					:   Any								= GPTaTokenizerFast.from_pretrained('''gpt2'''      )

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request'''      ,						return_value=_A      ) as mock_head:
            lowercase					:   Dict								= GPTaTokenizerFast.from_pretrained('''gpt2'''      )
            # This check we did call the fake head request
            mock_head.assert_called()






  def 					__a			(  self		:	str      )		->    Optional[int]:






       """simple docstring"""







       try:
            lowercase					:   Any								= tempfile.mktemp()
            with open(_A      ,						'''wb'''      ) as f:
                 http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      ,						_A      )

            lowercase					:   Union[str, Any]								= AlbertTokenizer.from_pretrained(_A      )
       finally:
            os.remove(_A      )

       # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
       # the current folder and have the right name.
       if os.path.isfile('''tokenizer.json'''      ):
            # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
            return
       try:
            with open('''tokenizer.json'''      ,						'''wb'''      ) as f:
                 http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''      ,						_A      )
            lowercase					:   Optional[Any]								= AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''      )
            # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
            self.assertEqual(tokenizer.vocab_size      ,						1_000      )
            # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

       finally:
            os.remove('''tokenizer.json'''      )






  def 					__a			(  self		:	Any      )		->    Optional[Any]:






       """simple docstring"""







       lowercase					:   Optional[Any]								= AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      )


@is_staging_test
class 						_A					(  unittest.TestCase      ):
  _UpperCamelCase   :			Dict =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






  @classmethod
  def 					__a			(  cls		:	List[Any]      )		->    str:






       """simple docstring"""







       lowercase					:   Union[str, Any]								= TOKEN
       HfFolder.save_token(_A      )






  @classmethod
  def 					__a			(  cls		:	int      )		->    Optional[Any]:






       """simple docstring"""







       try:
            delete_repo(token=cls._token      ,						repo_id='''test-tokenizer'''      )
       except HTTPError:
            pass

       try:
            delete_repo(token=cls._token      ,						repo_id='''valid_org/test-tokenizer-org'''      )
       except HTTPError:
            pass

       try:
            delete_repo(token=cls._token      ,						repo_id='''test-dynamic-tokenizer'''      )
       except HTTPError:
            pass






  def 					__a			(  self		:	Optional[int]      )		->    Optional[Any]:






       """simple docstring"""







       with tempfile.TemporaryDirectory() as tmp_dir:
            lowercase					:   Any								= os.path.join(_A      ,						'''vocab.txt'''      )
            with open(_A      ,						'''w'''      ,						encoding='''utf-8'''      ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
            lowercase					:   Optional[Any]								= BertTokenizer(_A      )

       tokenizer.push_to_hub('''test-tokenizer'''      ,						use_auth_token=self._token      )
       lowercase					:   Optional[Any]								= BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""      )
       self.assertDictEqual(new_tokenizer.vocab      ,						tokenizer.vocab      )

       # Reset repo
       delete_repo(token=self._token      ,						repo_id='''test-tokenizer'''      )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(_A      ,						repo_id='''test-tokenizer'''      ,						push_to_hub=_A      ,						use_auth_token=self._token      )

       lowercase					:   Any								= BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""      )
       self.assertDictEqual(new_tokenizer.vocab      ,						tokenizer.vocab      )






  def 					__a			(  self		:	List[Any]      )		->    List[Any]:






       """simple docstring"""







       with tempfile.TemporaryDirectory() as tmp_dir:
            lowercase					:   Optional[Any]								= os.path.join(_A      ,						'''vocab.txt'''      )
            with open(_A      ,						'''w'''      ,						encoding='''utf-8'''      ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
            lowercase					:   str								= BertTokenizer(_A      )

       tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''      ,						use_auth_token=self._token      )
       lowercase					:   int								= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
       self.assertDictEqual(new_tokenizer.vocab      ,						tokenizer.vocab      )

       # Reset repo
       delete_repo(token=self._token      ,						repo_id='''valid_org/test-tokenizer-org'''      )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(
                _A      ,						repo_id='''valid_org/test-tokenizer-org'''      ,						push_to_hub=_A      ,						use_auth_token=self._token      )

       lowercase					:   Optional[Any]								= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''      )
       self.assertDictEqual(new_tokenizer.vocab      ,						tokenizer.vocab      )






  @require_tokenizers
  def 					__a			(  self		:	List[Any]      )		->    Dict:






       """simple docstring"""







       CustomTokenizer.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
            lowercase					:   Tuple								= os.path.join(_A      ,						'''vocab.txt'''      )
            with open(_A      ,						'''w'''      ,						encoding='''utf-8'''      ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )
            lowercase					:   List[Any]								= CustomTokenizer(_A      )

       # No fast custom tokenizer
       tokenizer.push_to_hub('''test-dynamic-tokenizer'''      ,						use_auth_token=self._token      )

       lowercase					:   Optional[int]								= AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""      ,						trust_remote_code=_A      )
       # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__      ,						'''CustomTokenizer'''      )

       # Fast and slow custom tokenizer
       CustomTokenizerFast.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
            lowercase					:   Union[str, Any]								= os.path.join(_A      ,						'''vocab.txt'''      )
            with open(_A      ,						'''w'''      ,						encoding='''utf-8'''      ) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]      )      )

            lowercase					:   Optional[Any]								= BertTokenizerFast.from_pretrained(_A      )
            bert_tokenizer.save_pretrained(_A      )
            lowercase					:   List[str]								= CustomTokenizerFast.from_pretrained(_A      )

       tokenizer.push_to_hub('''test-dynamic-tokenizer'''      ,						use_auth_token=self._token      )

       lowercase					:   Union[str, Any]								= AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""      ,						trust_remote_code=_A      )
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__      ,						'''CustomTokenizerFast'''      )
       lowercase					:   Tuple								= AutoTokenizer.from_pretrained(
           f"""{USER}/test-dynamic-tokenizer"""      ,						use_fast=_A      ,						trust_remote_code=_A      )
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__      ,						'''CustomTokenizer'''      )


class 						_A					(  unittest.TestCase      ):






  def 					__a			(  self		:	Optional[Any]      )		->    Dict:






       """simple docstring"""







       lowercase					:   Optional[Any]								= Trie()
       trie.add('''Hello 友達'''      )
       self.assertEqual(trie.data      ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )
       trie.add('''Hello'''      )
       trie.data
       self.assertEqual(trie.data      ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}      )






  def 					__a			(  self		:	List[Any]      )		->    List[Any]:






       """simple docstring"""







       lowercase					:   Union[str, Any]								= Trie()
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )      ,						['''[CLS] This is a extra_id_100''']      )
       trie.add('''[CLS]'''      )
       trie.add('''extra_id_1'''      )
       trie.add('''extra_id_100'''      )
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''      )      ,						['''[CLS]''', ''' This is a ''', '''extra_id_100''']      )






  def 					__a			(  self		:	Union[str, Any]      )		->    Union[str, Any]:






       """simple docstring"""







       lowercase					:   int								= Trie()
       trie.add('''A'''      )
       self.assertEqual(trie.split('''ABC'''      )      ,						['''A''', '''BC''']      )
       self.assertEqual(trie.split('''BCA'''      )      ,						['''BC''', '''A''']      )






  def 					__a			(  self		:	Union[str, Any]      )		->    Tuple:






       """simple docstring"""







       lowercase					:   Tuple								= Trie()
       trie.add('''TOKEN]'''      )
       trie.add('''[SPECIAL_TOKEN]'''      )
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )      ,						['''This is something ''', '''[SPECIAL_TOKEN]''']      )






  def 					__a			(  self		:	Optional[Any]      )		->    Optional[Any]:






       """simple docstring"""







       lowercase					:   int								= Trie()
       trie.add('''A'''      )
       trie.add('''P'''      )
       trie.add('''[SPECIAL_TOKEN]'''      )
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''      )      ,						['''This is something ''', '''[SPECIAL_TOKEN]''']      )






  def 					__a			(  self		:	Optional[int]      )		->    Dict:






       """simple docstring"""







       lowercase					:   int								= Trie()
       trie.add('''AB'''      )
       trie.add('''B'''      )
       trie.add('''C'''      )
       self.assertEqual(trie.split('''ABC'''      )      ,						['''AB''', '''C''']      )






  def 					__a			(  self		:	List[Any]      )		->    Tuple:






       """simple docstring"""







       lowercase					:   List[Any]								= Trie()
       trie.add('''ABC'''      )
       trie.add('''B'''      )
       trie.add('''CD'''      )
       self.assertEqual(trie.split('''ABCD'''      )      ,						['''ABC''', '''D''']      )






  def 					__a			(  self		:	Optional[Any]      )		->    Tuple:






       """simple docstring"""







       lowercase					:   List[str]								= Trie()
       lowercase					:   Optional[int]								= trie.cut_text('''ABC'''      ,						[0, 0, 2, 1, 2, 3]      )
       self.assertEqual(_A      ,						['''AB''', '''C''']      )
'''simple docstring'''




from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_tokenizers_available, is_torch_available
from ...utils import OptionalDependencyNotAvailable


__SCREAMING_SNAKE_CASE       :     Optional[int]     =  {"""configuration_gpt_neox""": ["""GPT_NEOX_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTNeoXConfig"""]}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __SCREAMING_SNAKE_CASE       :     Tuple     =  ["""GPTNeoXTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __SCREAMING_SNAKE_CASE       :     Dict     =  [
      """GPT_NEOX_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """GPTNeoXForCausalLM""",
      """GPTNeoXForQuestionAnswering""",
      """GPTNeoXForSequenceClassification""",
      """GPTNeoXForTokenClassification""",
      """GPTNeoXLayer""",
      """GPTNeoXModel""",
      """GPTNeoXPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_gpt_neox import GPT_NEOX_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXConfig

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_gpt_neox_fast import GPTNeoXTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_gpt_neox import (
        GPT_NEOX_PRETRAINED_MODEL_ARCHIVE_LIST,
        GPTNeoXForCausalLM,
        GPTNeoXForQuestionAnswering,
        GPTNeoXForSequenceClassification,
        GPTNeoXForTokenClassification,
        GPTNeoXLayer,
        GPTNeoXModel,
        GPTNeoXPreTrainedModel,
    )


else:
  import sys

  __SCREAMING_SNAKE_CASE       :     Any     =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

