






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast






class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						Dict  )    ->str:




       """simple docstring"""


       a		  =      mock.Mock()
       a		  =      500
       a		  =      {}
       a		  =      HTTPError
       a		  =      {}

       # Download this model to make sure it's in the cache.
       a		  =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''  )

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request''' ,					return_value=__UpperCAmelCase  ) as mock_head:
        a		  =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''  )
        # This check we did call the fake head request
        mock_head.assert_called()







      @require_tokenizers
      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Dict:




       """simple docstring"""


       a		  =      mock.Mock()
       a		  =      500
       a		  =      {}
       a		  =      HTTPError
       a		  =      {}

       # Download this model to make sure it's in the cache.
       a		  =      GPTaTokenizerFast.from_pretrained('''gpt2'''  )

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('''requests.Session.request''' ,					return_value=__UpperCAmelCase  ) as mock_head:
        a		  =      GPTaTokenizerFast.from_pretrained('''gpt2'''  )
        # This check we did call the fake head request
        mock_head.assert_called()







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Union[str, Any]:




       """simple docstring"""


       try:
        a		  =      tempfile.mktemp()
        with open(__UpperCAmelCase ,					'''wb'''  ) as f:
         http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''' ,					__UpperCAmelCase  )

        a		  =      AlbertTokenizer.from_pretrained(__UpperCAmelCase  )
       finally:
        os.remove(__UpperCAmelCase  )

       # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
       # the current folder and have the right name.
       if os.path.isfile('''tokenizer.json'''  ):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
       try:
        with open('''tokenizer.json''' ,					'''wb'''  ) as f:
         http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''' ,					__UpperCAmelCase  )
        a		  =      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''  )
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size ,					1_000  )
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

       finally:
        os.remove('''tokenizer.json'''  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->Optional[int]:




       """simple docstring"""


       a		  =      AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''  )






@is_staging_test
class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      __snake_case       =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']







      @classmethod
      def       __lowerCAmelCase					(      cls      :						Optional[Any]  )    ->Dict:




       """simple docstring"""


       a		  =      TOKEN
       HfFolder.save_token(__UpperCAmelCase  )







      @classmethod
      def       __lowerCAmelCase					(      cls      :						str  )    ->Optional[Any]:




       """simple docstring"""


       try:
        delete_repo(token=cls._token ,					repo_id='''test-tokenizer'''  )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token ,					repo_id='''valid_org/test-tokenizer-org'''  )
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token ,					repo_id='''test-dynamic-tokenizer'''  )
       except HTTPError:
        pass







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->Optional[int]:




       """simple docstring"""


       with tempfile.TemporaryDirectory() as tmp_dir:
        a		  =      os.path.join(__UpperCAmelCase ,					'''vocab.txt'''  )
        with open(__UpperCAmelCase ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]  )  )
        a		  =      BertTokenizer(__UpperCAmelCase  )

       tokenizer.push_to_hub('''test-tokenizer''' ,					use_auth_token=self._token  )
       a		  =      BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
       self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab  )

       # Reset repo
       delete_repo(token=self._token ,					repo_id='''test-tokenizer'''  )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__UpperCAmelCase ,					repo_id='''test-tokenizer''' ,					push_to_hub=__UpperCAmelCase ,					use_auth_token=self._token  )

       a		  =      BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
       self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab  )







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Tuple:




       """simple docstring"""


       with tempfile.TemporaryDirectory() as tmp_dir:
        a		  =      os.path.join(__UpperCAmelCase ,					'''vocab.txt'''  )
        with open(__UpperCAmelCase ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]  )  )
        a		  =      BertTokenizer(__UpperCAmelCase  )

       tokenizer.push_to_hub('''valid_org/test-tokenizer-org''' ,					use_auth_token=self._token  )
       a		  =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''  )
       self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab  )

       # Reset repo
       delete_repo(token=self._token ,					repo_id='''valid_org/test-tokenizer-org'''  )

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            __UpperCAmelCase ,					repo_id='''valid_org/test-tokenizer-org''' ,					push_to_hub=__UpperCAmelCase ,					use_auth_token=self._token  )

       a		  =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''  )
       self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab  )







      @require_tokenizers
      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->List[str]:




       """simple docstring"""


       CustomTokenizer.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
        a		  =      os.path.join(__UpperCAmelCase ,					'''vocab.txt'''  )
        with open(__UpperCAmelCase ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]  )  )
        a		  =      CustomTokenizer(__UpperCAmelCase  )

       # No fast custom tokenizer
       tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,					use_auth_token=self._token  )

       a		  =      AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer""" ,					trust_remote_code=__UpperCAmelCase  )
       # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__ ,					'''CustomTokenizer'''  )

       # Fast and slow custom tokenizer
       CustomTokenizerFast.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
        a		  =      os.path.join(__UpperCAmelCase ,					'''vocab.txt'''  )
        with open(__UpperCAmelCase ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]  )  )

        a		  =      BertTokenizerFast.from_pretrained(__UpperCAmelCase  )
        bert_tokenizer.save_pretrained(__UpperCAmelCase  )
        a		  =      CustomTokenizerFast.from_pretrained(__UpperCAmelCase  )

       tokenizer.push_to_hub('''test-dynamic-tokenizer''' ,					use_auth_token=self._token  )

       a		  =      AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer""" ,					trust_remote_code=__UpperCAmelCase  )
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__ ,					'''CustomTokenizerFast'''  )
       a		  =      AutoTokenizer.from_pretrained(
           F"""{USER}/test-dynamic-tokenizer""" ,					use_fast=__UpperCAmelCase ,					trust_remote_code=__UpperCAmelCase  )
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__ ,					'''CustomTokenizer'''  )






class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						Any  )    ->Tuple:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''Hello 友達'''  )
       self.assertEqual(trie.data ,					{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}  )
       trie.add('''Hello'''  )
       trie.data
       self.assertEqual(trie.data ,					{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->Optional[int]:




       """simple docstring"""


       a		  =      Trie()
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''  ) ,					['''[CLS] This is a extra_id_100''']  )
       trie.add('''[CLS]'''  )
       trie.add('''extra_id_1'''  )
       trie.add('''extra_id_100'''  )
       self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''  ) ,					['''[CLS]''', ''' This is a ''', '''extra_id_100''']  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Optional[int]:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''A'''  )
       self.assertEqual(trie.split('''ABC'''  ) ,					['''A''', '''BC''']  )
       self.assertEqual(trie.split('''BCA'''  ) ,					['''BC''', '''A''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->Tuple:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''TOKEN]'''  )
       trie.add('''[SPECIAL_TOKEN]'''  )
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''  ) ,					['''This is something ''', '''[SPECIAL_TOKEN]''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->int:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''A'''  )
       trie.add('''P'''  )
       trie.add('''[SPECIAL_TOKEN]'''  )
       self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''  ) ,					['''This is something ''', '''[SPECIAL_TOKEN]''']  )







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->str:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''AB'''  )
       trie.add('''B'''  )
       trie.add('''C'''  )
       self.assertEqual(trie.split('''ABC'''  ) ,					['''AB''', '''C''']  )







      def       __lowerCAmelCase					(      self      :						Dict  )    ->int:




       """simple docstring"""


       a		  =      Trie()
       trie.add('''ABC'''  )
       trie.add('''B'''  )
       trie.add('''CD'''  )
       self.assertEqual(trie.split('''ABCD'''  ) ,					['''ABC''', '''D''']  )







      def       __lowerCAmelCase					(      self      :						str  )    ->Optional[int]:




       """simple docstring"""


       a		  =      Trie()
       a		  =      trie.cut_text('''ABC''' ,					[0, 0, 2, 1, 2, 3]  )
       self.assertEqual(__UpperCAmelCase ,					['''AB''', '''C''']  )










def 						_a    (      a :int )							->			list:

 # bit count represents no. of bits in the gray code
 if bit_count < 0:
  raise ValueError('''The given input must be positive''' )

 # get the generated string sequence
 a		  =      gray_code_sequence_string(a )
 #
 # convert them to integers
 for i in range(len(a ) ):
  a		  =      int(sequence[i]	,							2 )

 return sequence
def 						_a    (      a :int )							->			list:

 # The approach is a recursive one
 # Base case achieved when either n = 0 or n=1
 if bit_count == 0:
  return ["0"]

 if bit_count == 1:
  return ["0", "1"]

 a		  =      1 << bit_count  # defines the length of the sequence
 # 1<< n is equivalent to 2^n

 # recursive answer will generate answer for n-1 bits
 a		  =      gray_code_sequence_string(bit_count - 1 )

 a		  =      []

 # append 0 to first half of the smaller sequence generated
 for i in range(seq_len // 2 ):
  a		  =      '''0''' + smaller_sequence[i]
  sequence.append(a )

 # append 1 to second half ... start from the end of the list
 for i in reversed(range(seq_len // 2 ) ):
  a		  =      '''1''' + smaller_sequence[i]
  sequence.append(a )

 return sequence


if __name__ == "__main__":
     import doctest

     doctest.testmod()


