





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			CLIPTokenizer
 lowercase_		=			CLIPTokenizerFast
 lowercase_		=			True
 lowercase_		=			{}
 lowercase_		=			False

 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->Union[str, Any]:



   '''simple docstring'''



   super().setUp()

   # fmt: off
   lowerCamelCase__:      str			  =["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
   # fmt: on
   lowerCamelCase__:      Optional[int]			  =dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_))))
   lowerCamelCase__:      Dict			  =["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
   lowerCamelCase__:      List[str]			  ={"unk_token": "<unk>"}

   lowerCamelCase__:      Dict			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["vocab_file"])
   lowerCamelCase__:      Dict			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["merges_file"])
   with open(self.vocab_file       , "w"       , encoding="utf-8") as fp:
     fp.write(json.dumps(UpperCAmelCase_) + "\n")
   with open(self.merges_file       , "w"       , encoding="utf-8") as fp:
     fp.write("\n".join(UpperCAmelCase_))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int]       , **UpperCAmelCase_				:   List[str])       ->List[Any]:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return CLIPTokenizer.from_pretrained(self.tmpdirname       , **UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any       , **UpperCAmelCase_				:   Any)       ->Dict:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return CLIPTokenizerFast.from_pretrained(self.tmpdirname       , **UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   int       , UpperCAmelCase_				:   List[Any])       ->int:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  ="lower newer"
   lowerCamelCase__:      Optional[Any]			  ="lower newer"
   return input_text, output_text

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      Any			  =CLIPTokenizer(self.vocab_file       , self.merges_file       , **self.special_tokens_map)
   lowerCamelCase__:      Any			  ="lower newer"
   lowerCamelCase__:      List[str]			  =["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
   lowerCamelCase__:      List[str]			  =tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Tuple			  =tokens + [tokenizer.unk_token]
   lowerCamelCase__:      List[str]			  =[10, 2, 16, 9, 3, 2, 16, 20]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , UpperCAmelCase_)

 @require_ftfy
 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->List[Any]:



   '''simple docstring'''



   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
       lowerCamelCase__:      Tuple			  =self.tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)
       lowerCamelCase__:      int			  =self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)

       lowerCamelCase__:      Union[str, Any]			  ="A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
       lowerCamelCase__:      str			  =tokenizer_s.tokenize(UpperCAmelCase_)
       lowerCamelCase__:      int			  =tokenizer_r.tokenize(UpperCAmelCase_)

       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

       # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
       # with Tilde) encoded in 2 different ways
       lowerCamelCase__:      Optional[int]			  ="xa\u0303y" + " " + "x\xe3y"
       lowerCamelCase__:      Union[str, Any]			  =tokenizer_s.tokenize(UpperCAmelCase_)
       lowerCamelCase__:      Tuple			  =tokenizer_r.tokenize(UpperCAmelCase_)

       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

       # Test that the tokenization is identical on unicode of space type
       lowerCamelCase__:      Tuple			  =[
           "\u0009",  # (horizontal tab, '\t')
           "\u000B",  # (vertical tab)
           "\u000C",  # (form feed)
           "\u0020",  # (space, ' ')
           "\u200E",  # (left-to-right mark):w
           "\u200F",  # (right-to-left mark)
       ]
       for unicode_seq in spaces_unicodes:
         lowerCamelCase__:      int			  =tokenizer_s.tokenize(UpperCAmelCase_)
         lowerCamelCase__:      Optional[Any]			  =tokenizer_r.tokenize(UpperCAmelCase_)

         self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

       # Test that the tokenization is identical on unicode of line break type
       lowerCamelCase__:      Tuple			  =[
           "\u000A",  # (line feed, '\n')
           "\r\n",  # (carriage return and line feed, '\r\n')
           "\u000D",  # (carriage return, '\r')
           "\r",  # (carriage return, '\r')
           "\u000D",  # (carriage return, '\r')
           "\u2028",  # (line separator)
           "\u2029",  # (paragraph separator)
           # "\u0085", # (next line)
       ]

       # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
       # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
       # space (and thus into an empty list).

       for unicode_seq in line_break_unicodes:
         lowerCamelCase__:      List[str]			  =tokenizer_s.tokenize(UpperCAmelCase_)
         lowerCamelCase__:      str			  =tokenizer_r.tokenize(UpperCAmelCase_)

         self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->Optional[int]:



   '''simple docstring'''



   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
       lowerCamelCase__:      List[str]			  ="hello"  # `hello` is a token in the vocabulary of `pretrained_name`
       lowerCamelCase__:      str			  =F"""{text_of_1_token} {text_of_1_token}"""

       lowerCamelCase__:      int			  =self.rust_tokenizer_class.from_pretrained(
           UpperCAmelCase_       , use_fast=UpperCAmelCase_       , )
       lowerCamelCase__:      str			  =tokenizer_r(UpperCAmelCase_       , return_offsets_mapping=UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
       self.assertEqual(encoding.offset_mapping[0]       , (0, len(UpperCAmelCase_)))
       self.assertEqual(
           encoding.offset_mapping[1]       , (len(UpperCAmelCase_) + 1, len(UpperCAmelCase_) + 1 + len(UpperCAmelCase_))       , )

       lowerCamelCase__:      str			  =F""" {text}"""

       lowerCamelCase__:      Optional[Any]			  =self.rust_tokenizer_class.from_pretrained(
           UpperCAmelCase_       , use_fast=UpperCAmelCase_       , )
       lowerCamelCase__:      Tuple			  =tokenizer_r(UpperCAmelCase_       , return_offsets_mapping=UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
       self.assertEqual(encoding.offset_mapping[0]       , (1, 1 + len(UpperCAmelCase_)))
       self.assertEqual(
           encoding.offset_mapping[1]       , (1 + len(UpperCAmelCase_) + 1, 1 + len(UpperCAmelCase_) + 1 + len(UpperCAmelCase_))       , )

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Dict:



   '''simple docstring'''



   with self.assertRaises(UpperCAmelCase_) as context:
     self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer")

   self.assertTrue(
       context.exception.args[0].startswith(
           "The `backend_tokenizer` provided does not match the expected format."))

 @require_ftfy
 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->List[Any]:



   '''simple docstring'''



   super().test_tokenization_python_rust_equals()







 def 							SCREAMING_SNAKE_CASE_					(self				:   str)       ->Union[str, Any]:



   '''simple docstring'''



   pass








import logging

from transformers.configuration_utils import PretrainedConfig


__A									=				logging.getLogger(__name__)



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"masked_bert"

 def __init__(self				:   Dict       , UpperCAmelCase_				:   Any=30_522       , UpperCAmelCase_				:   List[Any]=768       , UpperCAmelCase_				:   Optional[Any]=12       , UpperCAmelCase_				:   str=12       , UpperCAmelCase_				:   Tuple=3_072       , UpperCAmelCase_				:   str="gelu"       , UpperCAmelCase_				:   Any=0.1       , UpperCAmelCase_				:   Tuple=0.1       , UpperCAmelCase_				:   Optional[Any]=512       , UpperCAmelCase_				:   Union[str, Any]=2       , UpperCAmelCase_				:   str=0.02       , UpperCAmelCase_				:   str=1E-1_2       , UpperCAmelCase_				:   Union[str, Any]=0       , UpperCAmelCase_				:   str="topK"       , UpperCAmelCase_				:   List[str]="constant"       , UpperCAmelCase_				:   str=0.0       , **UpperCAmelCase_				:   int       , )       ->List[Any]:



   '''simple docstring'''



   super().__init__(pad_token_id=UpperCAmelCase_       , **UpperCAmelCase_)

   lowerCamelCase__:      Optional[int]			  =vocab_size
   lowerCamelCase__:      Dict			  =hidden_size
   lowerCamelCase__:      Optional[int]			  =num_hidden_layers
   lowerCamelCase__:      Any			  =num_attention_heads
   lowerCamelCase__:      List[Any]			  =hidden_act
   lowerCamelCase__:      str			  =intermediate_size
   lowerCamelCase__:      Dict			  =hidden_dropout_prob
   lowerCamelCase__:      str			  =attention_probs_dropout_prob
   lowerCamelCase__:      int			  =max_position_embeddings
   lowerCamelCase__:      Tuple			  =type_vocab_size
   lowerCamelCase__:      str			  =initializer_range
   lowerCamelCase__:      List[Any]			  =layer_norm_eps
   lowerCamelCase__:      str			  =pruning_method
   lowerCamelCase__:      Union[str, Any]			  =mask_init
   lowerCamelCase__:      Optional[Any]			  =mask_scale

