





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			MobileBertTokenizer
 lowercase_		=			MobileBertTokenizerFast
 lowercase_		=			True
 lowercase_		=			True
 lowercase_		=			filter_non_english
 lowercase_		=			"google/mobilebert-uncased"

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Any:



   '''simple docstring'''



   super().setUp()

   lowerCamelCase__:      Any			  =[
       "[UNK]",
       "[CLS]",
       "[SEP]",
       "[PAD]",
       "[MASK]",
       "want",
       "##want",
       "##ed",
       "wa",
       "un",
       "runn",
       "##ing",
       ",",
       "low",
       "lowest",
   ]
   lowerCamelCase__:      Optional[int]			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["vocab_file"])
   with open(self.vocab_file       , "w"       , encoding="utf-8") as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

   lowerCamelCase__:      str			  =[
       (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
       for tokenizer_def in self.tokenizers_list
   ]

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[str]       , UpperCAmelCase_				:   int)       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      List[str]			  ="UNwant\u00E9d,running"
   lowerCamelCase__:      Dict			  ="unwanted, running"
   return input_text, output_text

 def 							SCREAMING_SNAKE_CASE_					(self				:   int)       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  =self.tokenizer_class(self.vocab_file)

   lowerCamelCase__:      Union[str, Any]			  =tokenizer.tokenize("UNwant\u00E9d,running")
   self.assertListEqual(UpperCAmelCase_       , ["un", "##want", "##ed", ",", "runn", "##ing"])
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , [9, 6, 7, 12, 10, 11])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Optional[Any]:



   '''simple docstring'''



   if not self.test_rust_tokenizer:
     return

   lowerCamelCase__:      Tuple			  =self.get_tokenizer()
   lowerCamelCase__:      Union[str, Any]			  =self.get_rust_tokenizer()

   lowerCamelCase__:      List[Any]			  ="UNwant\u00E9d,running"

   lowerCamelCase__:      Optional[Any]			  =tokenizer.tokenize(UpperCAmelCase_)
   lowerCamelCase__:      List[str]			  =rust_tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Optional[int]			  =tokenizer.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
   lowerCamelCase__:      int			  =rust_tokenizer.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      List[str]			  =self.get_rust_tokenizer()
   lowerCamelCase__:      Any			  =tokenizer.encode(UpperCAmelCase_)
   lowerCamelCase__:      int			  =rust_tokenizer.encode(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   # With lower casing
   lowerCamelCase__:      Dict			  =self.get_tokenizer(do_lower_case=UpperCAmelCase_)
   lowerCamelCase__:      List[str]			  =self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_)

   lowerCamelCase__:      Optional[int]			  ="UNwant\u00E9d,running"

   lowerCamelCase__:      Tuple			  =tokenizer.tokenize(UpperCAmelCase_)
   lowerCamelCase__:      List[str]			  =rust_tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Union[str, Any]			  =tokenizer.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
   lowerCamelCase__:      Tuple			  =rust_tokenizer.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Union[str, Any]			  =self.get_rust_tokenizer()
   lowerCamelCase__:      Any			  =tokenizer.encode(UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =rust_tokenizer.encode(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   int)       ->Union[str, Any]:



   '''simple docstring'''



   lowerCamelCase__:      str			  =BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")       , ["ah", "\u535A", "\u63A8", "zz"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:      Union[str, Any]			  =BasicTokenizer(do_lower_case=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")       , ["hello", "!", "how", "are", "you", "?"])
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       , ["hello"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   int)       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =BasicTokenizer(do_lower_case=UpperCAmelCase_       , strip_accents=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       , ["hällo", "!", "how", "are", "you", "?"])
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       , ["h\u00E9llo"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  =BasicTokenizer(do_lower_case=UpperCAmelCase_       , strip_accents=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       , ["hallo", "!", "how", "are", "you", "?"])
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       , ["hello"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int])       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Any			  =BasicTokenizer(do_lower_case=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       , ["hallo", "!", "how", "are", "you", "?"])
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo")       , ["hello"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any])       ->Any:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  =BasicTokenizer(do_lower_case=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")       , ["HeLLo", "!", "how", "Are", "yoU", "?"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->int:



   '''simple docstring'''



   lowerCamelCase__:      str			  =BasicTokenizer(do_lower_case=UpperCAmelCase_       , strip_accents=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       , ["HäLLo", "!", "how", "Are", "yoU", "?"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int])       ->Tuple:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  =BasicTokenizer(do_lower_case=UpperCAmelCase_       , strip_accents=UpperCAmelCase_)

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")       , ["HaLLo", "!", "how", "Are", "yoU", "?"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      List[Any]			  =BasicTokenizer(do_lower_case=UpperCAmelCase_       , never_split=["[UNK]"])

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")       , ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any])       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

   lowerCamelCase__:      Optional[int]			  ={}
   for i, token in enumerate(UpperCAmelCase_):
     lowerCamelCase__:      List[str]			  =i
   lowerCamelCase__:      Optional[int]			  =WordpieceTokenizer(vocab=UpperCAmelCase_       , unk_token="[UNK]")

   self.assertListEqual(tokenizer.tokenize("")       , [])

   self.assertListEqual(tokenizer.tokenize("unwanted running")       , ["un", "##want", "##ed", "runn", "##ing"])

   self.assertListEqual(tokenizer.tokenize("unwantedX running")       , ["[UNK]", "runn", "##ing"])

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->List[str]:



   '''simple docstring'''



   self.assertTrue(_is_whitespace(" "))
   self.assertTrue(_is_whitespace("\t"))
   self.assertTrue(_is_whitespace("\r"))
   self.assertTrue(_is_whitespace("\n"))
   self.assertTrue(_is_whitespace("\u00A0"))

   self.assertFalse(_is_whitespace("A"))
   self.assertFalse(_is_whitespace("-"))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->int:



   '''simple docstring'''



   self.assertTrue(_is_control("\u0005"))

   self.assertFalse(_is_control("A"))
   self.assertFalse(_is_control(" "))
   self.assertFalse(_is_control("\t"))
   self.assertFalse(_is_control("\r"))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->List[str]:



   '''simple docstring'''



   self.assertTrue(_is_punctuation("-"))
   self.assertTrue(_is_punctuation("$"))
   self.assertTrue(_is_punctuation("`"))
   self.assertTrue(_is_punctuation("."))

   self.assertFalse(_is_punctuation("A"))
   self.assertFalse(_is_punctuation(" "))

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      List[str]			  =self.get_tokenizer()
   lowerCamelCase__:      Optional[Any]			  =self.get_rust_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_) for t in ["Test", "\xad", "test"]]       , [["[UNK]"], [], ["[UNK]"]])

   self.assertListEqual(
       [rust_tokenizer.tokenize(UpperCAmelCase_) for t in ["Test", "\xad", "test"]]       , [["[UNK]"], [], ["[UNK]"]])

 @slow
 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int])       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =self.tokenizer_class.from_pretrained("google/mobilebert-uncased")

   lowerCamelCase__:      Dict			  =tokenizer.encode("sequence builders"       , add_special_tokens=UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =tokenizer.encode("multi-sequence build"       , add_special_tokens=UpperCAmelCase_)

   lowerCamelCase__:      Union[str, Any]			  =tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_       , UpperCAmelCase_)

   assert encoded_sentence == [101] + text + [102]
   assert encoded_pair == [101] + text + [102] + text_a + [102]

 def 							SCREAMING_SNAKE_CASE_					(self				:   Any)       ->Tuple:



   '''simple docstring'''



   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
       lowerCamelCase__:      List[str]			  =self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)

       lowerCamelCase__:      Optional[Any]			  =F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
       lowerCamelCase__:      Any			  =tokenizer_r.encode_plus(
           UpperCAmelCase_       , return_attention_mask=UpperCAmelCase_       , return_token_type_ids=UpperCAmelCase_       , return_offsets_mapping=UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_       , )

       lowerCamelCase__:      Optional[int]			  =tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_       , "do_lower_case") else False
       lowerCamelCase__:      Optional[Any]			  =(
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "A"),
               ((1, 2), ","),
               ((3, 5), "na"),
               ((5, 6), "##ï"),
               ((6, 8), "##ve"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "Allen"),
               ((21, 23), "##NL"),
               ((23, 24), "##P"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "a"),
               ((1, 2), ","),
               ((3, 8), "naive"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "allen"),
               ((21, 23), "##nl"),
               ((23, 24), "##p"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]       , tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
       self.assertEqual([e[0] for e in expected_results]       , tokens["offset_mapping"])







 def 							SCREAMING_SNAKE_CASE_					(self				:   List[str])       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      str			  =["的", "人", "有"]
   lowerCamelCase__:      Optional[Any]			  ="".join(UpperCAmelCase_)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
       lowerCamelCase__:      str			  =True
       lowerCamelCase__:      List[str]			  =self.tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)
       lowerCamelCase__:      Tuple			  =self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)

       lowerCamelCase__:      List[Any]			  =tokenizer_p.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
       lowerCamelCase__:      Any			  =tokenizer_r.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)

       lowerCamelCase__:      List[str]			  =tokenizer_r.convert_ids_to_tokens(UpperCAmelCase_)
       lowerCamelCase__:      Dict			  =tokenizer_p.convert_ids_to_tokens(UpperCAmelCase_)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)
       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

       lowerCamelCase__:      Any			  =False
       lowerCamelCase__:      List[Any]			  =self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)
       lowerCamelCase__:      str			  =self.tokenizer_class.from_pretrained(UpperCAmelCase_       , **UpperCAmelCase_)

       lowerCamelCase__:      Dict			  =tokenizer_r.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)
       lowerCamelCase__:      List[Any]			  =tokenizer_p.encode(UpperCAmelCase_       , add_special_tokens=UpperCAmelCase_)

       lowerCamelCase__:      Dict			  =tokenizer_r.convert_ids_to_tokens(UpperCAmelCase_)
       lowerCamelCase__:      Dict			  =tokenizer_p.convert_ids_to_tokens(UpperCAmelCase_)

       # it is expected that only the first Chinese character is not preceded by "##".
       lowerCamelCase__:      str			  =[
           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase_)
       ]
       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)
       self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)





UpperCAmelCase__   	=							8.31_44_62  # Unit - J mol-1 K-1
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case     ,						__snake_case						)     ->			float:







 """simple docstring"""


 if moles < 0 or kelvin < 0 or volume < 0:
  raise ValueError('''Invalid inputs. Enter positive value.'''						)
 return moles * kelvin * UNIVERSAL_GAS_CONSTANT / volume
def 				UpperCAmelCase_					(							__snake_case     ,						__snake_case     ,						__snake_case						)     ->			float:







 """simple docstring"""


 if moles < 0 or kelvin < 0 or pressure < 0:
  raise ValueError('''Invalid inputs. Enter positive value.'''						)
 return moles * kelvin * UNIVERSAL_GAS_CONSTANT / pressure


if __name__ == "__main__":
  from doctest import testmod

  testmod()

