


'''simple docstring'''







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    lowercase_	(       lowerCAmelCase__    ,		unittest.TestCase  ):
    __UpperCAmelCase       =      ElectraTokenizer
    __UpperCAmelCase       =      ElectraTokenizerFast
    __UpperCAmelCase       =      True
    __UpperCAmelCase       =      True
    __UpperCAmelCase       =      filter_non_english


    def   __a				(	self       ):
          super().setUp()

          UpperCamelCase__      =							[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          UpperCamelCase__      =							os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES["vocab_file"]       )
          with open(self.vocab_file    ,   "w"    ,   encoding="utf-8"       ) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )


    def   __a				(	self    ,   a       ):
          UpperCamelCase__      =							"UNwant\u00E9d,running"
          UpperCamelCase__      =							"unwanted, running"
          return input_text, output_text


    def   __a				(	self       ):
          UpperCamelCase__      =							self.tokenizer_class(self.vocab_file       )

          UpperCamelCase__      =							tokenizer.tokenize("UNwant\u00E9d,running"       )
          self.assertListEqual(_A    ,   ["un", "##want", "##ed", ",", "runn", "##ing"]       )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_A       )    ,   [9, 6, 7, 12, 10, 11]       )


    def   __a				(	self       ):
          if not self.test_rust_tokenizer:
                return

          UpperCamelCase__      =							self.get_tokenizer()
          UpperCamelCase__      =							self.get_rust_tokenizer()

          UpperCamelCase__      =							"UNwant\u00E9d,running"

          UpperCamelCase__      =							tokenizer.tokenize(_A       )
          UpperCamelCase__      =							rust_tokenizer.tokenize(_A       )
          self.assertListEqual(_A    ,   _A       )

          UpperCamelCase__      =							tokenizer.encode(_A    ,   add_special_tokens=_A       )
          UpperCamelCase__      =							rust_tokenizer.encode(_A    ,   add_special_tokens=_A       )
          self.assertListEqual(_A    ,   _A       )

          UpperCamelCase__      =							self.get_rust_tokenizer()
          UpperCamelCase__      =							tokenizer.encode(_A       )
          UpperCamelCase__      =							rust_tokenizer.encode(_A       )
          self.assertListEqual(_A    ,   _A       )

          # With lower casing
          UpperCamelCase__      =							self.get_tokenizer(do_lower_case=_A       )
          UpperCamelCase__      =							self.get_rust_tokenizer(do_lower_case=_A       )

          UpperCamelCase__      =							"UNwant\u00E9d,running"

          UpperCamelCase__      =							tokenizer.tokenize(_A       )
          UpperCamelCase__      =							rust_tokenizer.tokenize(_A       )
          self.assertListEqual(_A    ,   _A       )

          UpperCamelCase__      =							tokenizer.encode(_A    ,   add_special_tokens=_A       )
          UpperCamelCase__      =							rust_tokenizer.encode(_A    ,   add_special_tokens=_A       )
          self.assertListEqual(_A    ,   _A       )

          UpperCamelCase__      =							self.get_rust_tokenizer()
          UpperCamelCase__      =							tokenizer.encode(_A       )
          UpperCamelCase__      =							rust_tokenizer.encode(_A       )
          self.assertListEqual(_A    ,   _A       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer()

          self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )    ,   ["ah", "\u535A", "\u63A8", "zz"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )    ,   ["hello", "!", "how", "are", "you", "?"]       )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )    ,   ["hello"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A    ,   strip_accents=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )    ,   ["hällo", "!", "how", "are", "you", "?"]       )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )    ,   ["h\u00E9llo"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A    ,   strip_accents=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )    ,   ["hallo", "!", "how", "are", "you", "?"]       )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )    ,   ["hello"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )    ,   ["hallo", "!", "how", "are", "you", "?"]       )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )    ,   ["hello"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )    ,   ["HeLLo", "!", "how", "Are", "yoU", "?"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A    ,   strip_accents=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )    ,   ["HäLLo", "!", "how", "Are", "yoU", "?"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A    ,   strip_accents=_A       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )    ,   ["HaLLo", "!", "how", "Are", "yoU", "?"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							BasicTokenizer(do_lower_case=_A    ,   never_split=["[UNK]"]       )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )    ,   ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

          UpperCamelCase__      =							{}
          for i, token in enumerate(_A       ):
                UpperCamelCase__      =							i
          UpperCamelCase__      =							WordpieceTokenizer(vocab=_A    ,   unk_token="[UNK]"       )

          self.assertListEqual(tokenizer.tokenize(""       )    ,   []       )

          self.assertListEqual(tokenizer.tokenize("unwanted running"       )    ,   ["un", "##want", "##ed", "runn", "##ing"]       )

          self.assertListEqual(tokenizer.tokenize("unwantedX running"       )    ,   ["[UNK]", "runn", "##ing"]       )


    def   __a				(	self       ):
          self.assertTrue(_is_whitespace(" "       )       )
          self.assertTrue(_is_whitespace("\t"       )       )
          self.assertTrue(_is_whitespace("\r"       )       )
          self.assertTrue(_is_whitespace("\n"       )       )
          self.assertTrue(_is_whitespace("\u00A0"       )       )

          self.assertFalse(_is_whitespace("A"       )       )
          self.assertFalse(_is_whitespace("-"       )       )


    def   __a				(	self       ):
          self.assertTrue(_is_control("\u0005"       )       )

          self.assertFalse(_is_control("A"       )       )
          self.assertFalse(_is_control(" "       )       )
          self.assertFalse(_is_control("\t"       )       )
          self.assertFalse(_is_control("\r"       )       )


    def   __a				(	self       ):
          self.assertTrue(_is_punctuation("-"       )       )
          self.assertTrue(_is_punctuation("$"       )       )
          self.assertTrue(_is_punctuation("`"       )       )
          self.assertTrue(_is_punctuation("."       )       )

          self.assertFalse(_is_punctuation("A"       )       )
          self.assertFalse(_is_punctuation(" "       )       )


    def   __a				(	self       ):
          UpperCamelCase__      =							self.get_tokenizer()
          UpperCamelCase__      =							self.get_rust_tokenizer()

          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(_A       ) for t in ["Test", "\xad", "test"]]    ,   [["[UNK]"], [], ["[UNK]"]]       )

          self.assertListEqual(
              [rust_tokenizer.tokenize(_A       ) for t in ["Test", "\xad", "test"]]    ,   [["[UNK]"], [], ["[UNK]"]]       )


    @slow
    def   __a				(	self       ):
          UpperCamelCase__      =							self.tokenizer_class.from_pretrained("google/electra-base-discriminator"       )

          UpperCamelCase__      =							tokenizer.encode("sequence builders"    ,   add_special_tokens=_A       )
          UpperCamelCase__      =							tokenizer.encode("multi-sequence build"    ,   add_special_tokens=_A       )

          UpperCamelCase__      =							tokenizer.build_inputs_with_special_tokens(_A       )
          UpperCamelCase__      =							tokenizer.build_inputs_with_special_tokens(_A    ,   _A       )

          assert encoded_sentence == [1_01] + text + [1_02]
          assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]


    def   __a				(	self       ):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                      UpperCamelCase__      =							self.rust_tokenizer_class.from_pretrained(_A    ,   **_A       )

                      UpperCamelCase__      =							f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                      UpperCamelCase__      =							tokenizer_r.encode_plus(
                          _A    ,   return_attention_mask=_A    ,   return_token_type_ids=_A    ,   return_offsets_mapping=_A    ,   add_special_tokens=_A    ,   )

                      UpperCamelCase__      =							tokenizer_r.do_lower_case if hasattr(_A    ,   "do_lower_case"       ) else False
                      UpperCamelCase__      =							(
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "A"),
                              ((1, 2), ","),
                              ((3, 5), "na"),
                              ((5, 6), "##ï"),
                              ((6, 8), "##ve"),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), "Allen"),
                              ((21, 23), "##NL"),
                              ((23, 24), "##P"),
                              ((25, 33), "sentence"),
                              ((33, 34), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "a"),
                              ((1, 2), ","),
                              ((3, 8), "naive"),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), "allen"),
                              ((21, 23), "##nl"),
                              ((23, 24), "##p"),
                              ((25, 33), "sentence"),
                              ((33, 34), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results]    ,   tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
                      self.assertEqual([e[0] for e in expected_results]    ,   tokens["offset_mapping"]       )


    def   __a				(	self       ):
          UpperCamelCase__      =							["的", "人", "有"]
          UpperCamelCase__      =							"".join(_A       )
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                      UpperCamelCase__      =							True
                      UpperCamelCase__      =							self.tokenizer_class.from_pretrained(_A    ,   **_A       )
                      UpperCamelCase__      =							self.rust_tokenizer_class.from_pretrained(_A    ,   **_A       )

                      UpperCamelCase__      =							tokenizer_p.encode(_A    ,   add_special_tokens=_A       )
                      UpperCamelCase__      =							tokenizer_r.encode(_A    ,   add_special_tokens=_A       )

                      UpperCamelCase__      =							tokenizer_r.convert_ids_to_tokens(_A       )
                      UpperCamelCase__      =							tokenizer_p.convert_ids_to_tokens(_A       )

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(_A    ,   _A       )
                      self.assertListEqual(_A    ,   _A       )

                      UpperCamelCase__      =							False
                      UpperCamelCase__      =							self.rust_tokenizer_class.from_pretrained(_A    ,   **_A       )
                      UpperCamelCase__      =							self.tokenizer_class.from_pretrained(_A    ,   **_A       )

                      UpperCamelCase__      =							tokenizer_r.encode(_A    ,   add_special_tokens=_A       )
                      UpperCamelCase__      =							tokenizer_p.encode(_A    ,   add_special_tokens=_A       )

                      UpperCamelCase__      =							tokenizer_r.convert_ids_to_tokens(_A       )
                      UpperCamelCase__      =							tokenizer_p.convert_ids_to_tokens(_A       )

                      # it is expected that only the first Chinese character is not preceded by "##".
                      UpperCamelCase__      =							[
                          f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_A       )
                      ]
                      self.assertListEqual(_A    ,   _A       )
                      self.assertListEqual(_A    ,   _A       )








def        _UpperCAmelCase							(	_SCREAMING_SNAKE_CASE       :		list							):





    """simple docstring"""


    if not isinstance(_SCREAMING_SNAKE_CASE			, _SCREAMING_SNAKE_CASE							):
        raise ValueError('Input series is not valid, valid series - [2, 4, 8]'							)
    if len(_SCREAMING_SNAKE_CASE							) == 0:
        raise ValueError('Input list must be a non empty list'							)
    if len(_SCREAMING_SNAKE_CASE							) == 1:
        return True
    try:
        SCREAMING_SNAKE_CASE_						=    series[1] / series[0]
        for index in range(len(_SCREAMING_SNAKE_CASE							) - 1							):
            if series[index + 1] / series[index] != common_ratio:
                return False
    except ZeroDivisionError:
        return False
    return True


def        _UpperCAmelCase							(	_SCREAMING_SNAKE_CASE       :		list							):





    """simple docstring"""


    if not isinstance(_SCREAMING_SNAKE_CASE			, _SCREAMING_SNAKE_CASE							):
        raise ValueError('Input series is not valid, valid series - [2, 4, 8]'							)
    if len(_SCREAMING_SNAKE_CASE							) == 0:
        raise ValueError('Input list must be a non empty list'							)
    SCREAMING_SNAKE_CASE_						=    1
    for value in series:
        answer *= value
    return pow(_SCREAMING_SNAKE_CASE			, 1 / len(_SCREAMING_SNAKE_CASE							)							)


if __name__ == "__main__":
       import doctest

       doctest.testmod()


