

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    __snake_case	(     lowerCAmelCase__  ,			unittest.TestCase							):
  __lowerCAmelCase  :						Optional[int]    			=	ElectraTokenizer
  __lowerCAmelCase  :						str    			=	ElectraTokenizerFast
  __lowerCAmelCase  :						List[Any]    			=	True
  __lowerCAmelCase  :						str    			=	True
  __lowerCAmelCase  :						Any    			=	filter_non_english

  def     lowerCAmelCase__  (       self):
      super().setUp()

      SCREAMING_SNAKE_CASE_						=    [
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      SCREAMING_SNAKE_CASE_						=    os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file    ,		'w'    ,		encoding='utf-8') as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

  def     lowerCAmelCase__  (       self    ,		_A):
      SCREAMING_SNAKE_CASE_						=    'UNwant\u00E9d,running'
      SCREAMING_SNAKE_CASE_						=    'unwanted, running'
      return input_text, output_text

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    self.tokenizer_class(self.vocab_file)

      SCREAMING_SNAKE_CASE_						=    tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(_A    ,		['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)    ,		[9, 6, 7, 12, 10, 11])

  def     lowerCAmelCase__  (       self):
      if not self.test_rust_tokenizer:
          return

      SCREAMING_SNAKE_CASE_						=    self.get_tokenizer()
      SCREAMING_SNAKE_CASE_						=    self.get_rust_tokenizer()

      SCREAMING_SNAKE_CASE_						=    'UNwant\u00E9d,running'

      SCREAMING_SNAKE_CASE_						=    tokenizer.tokenize(_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.tokenize(_A)
      self.assertListEqual(_A    ,		_A)

      SCREAMING_SNAKE_CASE_						=    tokenizer.encode(_A    ,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.encode(_A    ,		add_special_tokens=_A)
      self.assertListEqual(_A    ,		_A)

      SCREAMING_SNAKE_CASE_						=    self.get_rust_tokenizer()
      SCREAMING_SNAKE_CASE_						=    tokenizer.encode(_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.encode(_A)
      self.assertListEqual(_A    ,		_A)

      # With lower casing
      SCREAMING_SNAKE_CASE_						=    self.get_tokenizer(do_lower_case=_A)
      SCREAMING_SNAKE_CASE_						=    self.get_rust_tokenizer(do_lower_case=_A)

      SCREAMING_SNAKE_CASE_						=    'UNwant\u00E9d,running'

      SCREAMING_SNAKE_CASE_						=    tokenizer.tokenize(_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.tokenize(_A)
      self.assertListEqual(_A    ,		_A)

      SCREAMING_SNAKE_CASE_						=    tokenizer.encode(_A    ,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.encode(_A    ,		add_special_tokens=_A)
      self.assertListEqual(_A    ,		_A)

      SCREAMING_SNAKE_CASE_						=    self.get_rust_tokenizer()
      SCREAMING_SNAKE_CASE_						=    tokenizer.encode(_A)
      SCREAMING_SNAKE_CASE_						=    rust_tokenizer.encode(_A)
      self.assertListEqual(_A    ,		_A)

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')    ,		['ah', '\u535A', '\u63A8', 'zz'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,		['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,		['hello'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A    ,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,		['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,		['h\u00E9llo'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A    ,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,		['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,		['hello'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,		['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,		['hello'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A    ,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A    ,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    BasicTokenizer(do_lower_case=_A    ,		never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      SCREAMING_SNAKE_CASE_						=    {}
      for i, token in enumerate(_A):
          SCREAMING_SNAKE_CASE_						=    i
      SCREAMING_SNAKE_CASE_						=    WordpieceTokenizer(vocab=_A    ,		unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')    ,		[])

      self.assertListEqual(tokenizer.tokenize('unwanted running')    ,		['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')    ,		['[UNK]', 'runn', '##ing'])

  def     lowerCAmelCase__  (       self):
      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))

  def     lowerCAmelCase__  (       self):
      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))

  def     lowerCAmelCase__  (       self):
      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))

  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    self.get_tokenizer()
      SCREAMING_SNAKE_CASE_						=    self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_A) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(_A) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']])

  @slow
  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    self.tokenizer_class.from_pretrained('google/electra-base-discriminator')

      SCREAMING_SNAKE_CASE_						=    tokenizer.encode('sequence builders'    ,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_						=    tokenizer.encode('multi-sequence build'    ,		add_special_tokens=_A)

      SCREAMING_SNAKE_CASE_						=    tokenizer.build_inputs_with_special_tokens(_A)
      SCREAMING_SNAKE_CASE_						=    tokenizer.build_inputs_with_special_tokens(_A    ,		_A)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]

  def     lowerCAmelCase__  (       self):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
              SCREAMING_SNAKE_CASE_						=    self.rust_tokenizer_class.from_pretrained(_A    ,		**_A)

              SCREAMING_SNAKE_CASE_						=    f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
              SCREAMING_SNAKE_CASE_						=    tokenizer_r.encode_plus(
                  _A    ,		return_attention_mask=_A    ,		return_token_type_ids=_A    ,		return_offsets_mapping=_A    ,		add_special_tokens=_A    ,		)

              SCREAMING_SNAKE_CASE_						=    tokenizer_r.do_lower_case if hasattr(_A    ,		'do_lower_case') else False
              SCREAMING_SNAKE_CASE_						=    (
                  [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'A'),
                      ((1, 2), ','),
                      ((3, 5), 'na'),
                      ((5, 6), '##ï'),
                      ((6, 8), '##ve'),
                      ((9, 15), tokenizer_r.mask_token),
                      ((16, 21), 'Allen'),
                      ((21, 23), '##NL'),
                      ((23, 24), '##P'),
                      ((25, 33), 'sentence'),
                      ((33, 34), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
                  if not do_lower_case
                  else [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'a'),
                      ((1, 2), ','),
                      ((3, 8), 'naive'),
                      ((9, 15), tokenizer_r.mask_token),
                      ((16, 21), 'allen'),
                      ((21, 23), '##nl'),
                      ((23, 24), '##p'),
                      ((25, 33), 'sentence'),
                      ((33, 34), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
              )

              self.assertEqual(
                  [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
              self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping'])


  def     lowerCAmelCase__  (       self):
      SCREAMING_SNAKE_CASE_						=    ['的', '人', '有']
      SCREAMING_SNAKE_CASE_						=    ''.join(_A)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
              SCREAMING_SNAKE_CASE_						=    True
              SCREAMING_SNAKE_CASE_						=    self.tokenizer_class.from_pretrained(_A    ,		**_A)
              SCREAMING_SNAKE_CASE_						=    self.rust_tokenizer_class.from_pretrained(_A    ,		**_A)

              SCREAMING_SNAKE_CASE_						=    tokenizer_p.encode(_A    ,		add_special_tokens=_A)
              SCREAMING_SNAKE_CASE_						=    tokenizer_r.encode(_A    ,		add_special_tokens=_A)

              SCREAMING_SNAKE_CASE_						=    tokenizer_r.convert_ids_to_tokens(_A)
              SCREAMING_SNAKE_CASE_						=    tokenizer_p.convert_ids_to_tokens(_A)

              # it is expected that each Chinese character is not preceded by "##"
              self.assertListEqual(_A    ,		_A)
              self.assertListEqual(_A    ,		_A)

              SCREAMING_SNAKE_CASE_						=    False
              SCREAMING_SNAKE_CASE_						=    self.rust_tokenizer_class.from_pretrained(_A    ,		**_A)
              SCREAMING_SNAKE_CASE_						=    self.tokenizer_class.from_pretrained(_A    ,		**_A)

              SCREAMING_SNAKE_CASE_						=    tokenizer_r.encode(_A    ,		add_special_tokens=_A)
              SCREAMING_SNAKE_CASE_						=    tokenizer_p.encode(_A    ,		add_special_tokens=_A)

              SCREAMING_SNAKE_CASE_						=    tokenizer_r.convert_ids_to_tokens(_A)
              SCREAMING_SNAKE_CASE_						=    tokenizer_p.convert_ids_to_tokens(_A)

              # it is expected that only the first Chinese character is not preceded by "##".
              SCREAMING_SNAKE_CASE_						=    [
                  f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A)
              ]
              self.assertListEqual(_A    ,		_A)
              self.assertListEqual(_A    ,		_A)





def        _UpperCAmelCase							(	_SCREAMING_SNAKE_CASE       :		list							):





    """simple docstring"""


    if not isinstance(_SCREAMING_SNAKE_CASE			, _SCREAMING_SNAKE_CASE							):
        raise ValueError('Input series is not valid, valid series - [2, 4, 8]'							)
    if len(_SCREAMING_SNAKE_CASE							) == 0:
        raise ValueError('Input list must be a non empty list'							)
    if len(_SCREAMING_SNAKE_CASE							) == 1:
        return True
    try:
        SCREAMING_SNAKE_CASE_						=    series[1] / series[0]
        for index in range(len(_SCREAMING_SNAKE_CASE							) - 1							):
            if series[index + 1] / series[index] != common_ratio:
                return False
    except ZeroDivisionError:
        return False
    return True


def        _UpperCAmelCase							(	_SCREAMING_SNAKE_CASE       :		list							):





    """simple docstring"""


    if not isinstance(_SCREAMING_SNAKE_CASE			, _SCREAMING_SNAKE_CASE							):
        raise ValueError('Input series is not valid, valid series - [2, 4, 8]'							)
    if len(_SCREAMING_SNAKE_CASE							) == 0:
        raise ValueError('Input list must be a non empty list'							)
    SCREAMING_SNAKE_CASE_						=    1
    for value in series:
        answer *= value
    return pow(_SCREAMING_SNAKE_CASE			, 1 / len(_SCREAMING_SNAKE_CASE							)							)


if __name__ == "__main__":
       import doctest

       doctest.testmod()


