





'''simple docstring'''


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 		UpperCamelCase__     (       a	,	unittest.TestCase							):
      '''simple docstring'''






      _snake_case           =				ElectraTokenizer
      _snake_case           =				ElectraTokenizerFast
      _snake_case           =				True
      _snake_case           =				True
      _snake_case           =				filter_non_english




      def 	snake_case  (     self )				->  Optional[Any]:
        super().setUp()

        __lowerCAmelCase		:    str			     =  [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        __lowerCAmelCase		:    Any			     =  os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES['vocab_file'] )
        with open(self.vocab_file     , 'w'     , encoding='utf-8' ) as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  str:
        __lowerCAmelCase		:    Tuple			     =  'UNwant\u00E9d,running'
        __lowerCAmelCase		:    Optional[int]			     =  'unwanted, running'
        return input_text, output_text




      def 	snake_case  (     self )				->  List[str]:
        __lowerCAmelCase		:    Any			     =  self.tokenizer_class(self.vocab_file )

        __lowerCAmelCase		:    int			     =  tokenizer.tokenize('UNwant\u00E9d,running' )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , ['un', '##want', '##ed', ',', 'runn', '##ing'] )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE )     , [9, 6, 7, 12, 10, 11] )




      def 	snake_case  (     self )				->  str:
        if not self.test_rust_tokenizer:
          return

        __lowerCAmelCase		:    List[str]			     =  self.get_tokenizer()
        __lowerCAmelCase		:    Tuple			     =  self.get_rust_tokenizer()

        __lowerCAmelCase		:    Union[str, Any]			     =  'UNwant\u00E9d,running'

        __lowerCAmelCase		:    Optional[int]			     =  tokenizer.tokenize(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    int			     =  rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    Dict			     =  tokenizer.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Tuple			     =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    str			     =  self.get_rust_tokenizer()
        __lowerCAmelCase		:    Dict			     =  tokenizer.encode(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    List[str]			     =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        # With lower casing
        __lowerCAmelCase		:    Union[str, Any]			     =  self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    List[str]			     =  'UNwant\u00E9d,running'

        __lowerCAmelCase		:    Optional[Any]			     =  tokenizer.tokenize(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Tuple			     =  rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    List[str]			     =  tokenizer.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Tuple			     =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    List[str]			     =  self.get_rust_tokenizer()
        __lowerCAmelCase		:    List[str]			     =  tokenizer.encode(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Tuple			     =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE )
        self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )




      def 	snake_case  (     self )				->  int:
        __lowerCAmelCase		:    Dict			     =  BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )     , ['ah', '\u535A', '\u63A8', 'zz'] )




      def 	snake_case  (     self )				->  Optional[int]:
        __lowerCAmelCase		:    Any			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )     , ['hello', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )     , ['hello'] )




      def 	snake_case  (     self )				->  Union[str, Any]:
        __lowerCAmelCase		:    Tuple			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE     , strip_accents=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )     , ['hällo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )     , ['h\u00E9llo'] )




      def 	snake_case  (     self )				->  List[Any]:
        __lowerCAmelCase		:    List[str]			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE     , strip_accents=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )     , ['hallo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )     , ['hello'] )




      def 	snake_case  (     self )				->  Union[str, Any]:
        __lowerCAmelCase		:    List[Any]			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )     , ['hallo', '!', 'how', 'are', 'you', '?'] )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )     , ['hello'] )




      def 	snake_case  (     self )				->  Optional[Any]:
        __lowerCAmelCase		:    Tuple			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )     , ['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )




      def 	snake_case  (     self )				->  str:
        __lowerCAmelCase		:    List[str]			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE     , strip_accents=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )     , ['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )




      def 	snake_case  (     self )				->  int:
        __lowerCAmelCase		:    Union[str, Any]			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE     , strip_accents=SCREAMING_SNAKE_CASE )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )     , ['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )




      def 	snake_case  (     self )				->  List[str]:
        __lowerCAmelCase		:    Optional[int]			     =  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE     , never_split=['[UNK]'] )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )     , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )




      def 	snake_case  (     self )				->  Any:
        __lowerCAmelCase		:    int			     =  ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

        __lowerCAmelCase		:    Tuple			     =  {}
        for i, token in enumerate(SCREAMING_SNAKE_CASE ):
          __lowerCAmelCase		:    Union[str, Any]			     =  i
        __lowerCAmelCase		:    List[Any]			     =  WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE     , unk_token='[UNK]' )

        self.assertListEqual(tokenizer.tokenize('' )     , [] )

        self.assertListEqual(tokenizer.tokenize('unwanted running' )     , ['un', '##want', '##ed', 'runn', '##ing'] )

        self.assertListEqual(tokenizer.tokenize('unwantedX running' )     , ['[UNK]', 'runn', '##ing'] )




      def 	snake_case  (     self )				->  Tuple:
        self.assertTrue(_is_whitespace(' ' ) )
        self.assertTrue(_is_whitespace('\t' ) )
        self.assertTrue(_is_whitespace('\r' ) )
        self.assertTrue(_is_whitespace('\n' ) )
        self.assertTrue(_is_whitespace('\u00A0' ) )

        self.assertFalse(_is_whitespace('A' ) )
        self.assertFalse(_is_whitespace('-' ) )




      def 	snake_case  (     self )				->  str:
        self.assertTrue(_is_control('\u0005' ) )

        self.assertFalse(_is_control('A' ) )
        self.assertFalse(_is_control(' ' ) )
        self.assertFalse(_is_control('\t' ) )
        self.assertFalse(_is_control('\r' ) )




      def 	snake_case  (     self )				->  Dict:
        self.assertTrue(_is_punctuation('-' ) )
        self.assertTrue(_is_punctuation('$' ) )
        self.assertTrue(_is_punctuation('`' ) )
        self.assertTrue(_is_punctuation('.' ) )

        self.assertFalse(_is_punctuation('A' ) )
        self.assertFalse(_is_punctuation(' ' ) )




      def 	snake_case  (     self )				->  Union[str, Any]:
        __lowerCAmelCase		:    List[str]			     =  self.get_tokenizer()
        __lowerCAmelCase		:    int			     =  self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE ) for t in ['Test', '\xad', 'test']]     , [['[UNK]'], [], ['[UNK]']] )

        self.assertListEqual(
            [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE ) for t in ['Test', '\xad', 'test']]     , [['[UNK]'], [], ['[UNK]']] )




      @slow
      def 	snake_case  (     self )				->  Tuple:
        __lowerCAmelCase		:    str			     =  self.tokenizer_class.from_pretrained('google/electra-base-discriminator' )

        __lowerCAmelCase		:    str			     =  tokenizer.encode('sequence builders'     , add_special_tokens=SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Optional[int]			     =  tokenizer.encode('multi-sequence build'     , add_special_tokens=SCREAMING_SNAKE_CASE )

        __lowerCAmelCase		:    List[str]			     =  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Optional[Any]			     =  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

        assert encoded_sentence == [1_01] + text + [1_02]
        assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]




      def 	snake_case  (     self )				->  int:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
            __lowerCAmelCase		:    List[str]			     =  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    Optional[int]			     =  F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __lowerCAmelCase		:    int			     =  tokenizer_r.encode_plus(
                SCREAMING_SNAKE_CASE     , return_attention_mask=SCREAMING_SNAKE_CASE     , return_token_type_ids=SCREAMING_SNAKE_CASE     , return_offsets_mapping=SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE     , )

            __lowerCAmelCase		:    List[Any]			     =  tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE     , 'do_lower_case' ) else False
            __lowerCAmelCase		:    List[Any]			     =  (
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'A'),
                    ((1, 2), ','),
                    ((3, 5), 'na'),
                    ((5, 6), '##ï'),
                    ((6, 8), '##ve'),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), 'Allen'),
                    ((21, 23), '##NL'),
                    ((23, 24), '##P'),
                    ((25, 33), 'sentence'),
                    ((33, 34), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), 'a'),
                    ((1, 2), ','),
                    ((3, 8), 'naive'),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), 'allen'),
                    ((21, 23), '##nl'),
                    ((23, 24), '##p'),
                    ((25, 33), 'sentence'),
                    ((33, 34), '.'),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]     , tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
            self.assertEqual([e[0] for e in expected_results]     , tokens['offset_mapping'] )







      def 	snake_case  (     self )				->  Any:
        __lowerCAmelCase		:    List[str]			     =  ['的', '人', '有']
        __lowerCAmelCase		:    int			     =  ''.join(SCREAMING_SNAKE_CASE )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
            __lowerCAmelCase		:    List[Any]			     =  True
            __lowerCAmelCase		:    Dict			     =  self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    str			     =  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    Optional[Any]			     =  tokenizer_p.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    str			     =  tokenizer_r.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    Union[str, Any]			     =  tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    List[Any]			     =  tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE )

            # it is expected that each Chinese character is not preceded by "##"
            self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
            self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    List[str]			     =  False
            __lowerCAmelCase		:    Any			     =  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    Dict			     =  self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    List[str]			     =  tokenizer_r.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    Tuple			     =  tokenizer_p.encode(SCREAMING_SNAKE_CASE     , add_special_tokens=SCREAMING_SNAKE_CASE )

            __lowerCAmelCase		:    Optional[Any]			     =  tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE )
            __lowerCAmelCase		:    Tuple			     =  tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE )

            # it is expected that only the first Chinese character is not preceded by "##".
            __lowerCAmelCase		:    List[Any]			     =  [
                F"""##{token}""" if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE )
            ]
            self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
            self.assertListEqual(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )












'''simple docstring'''


from __future__ import annotations




def       A							(      _UpperCAmelCase   :				list					)  ->				int | float:


  '''simple docstring'''




  __lowerCAmelCase		:    List[Any]			     =  sorted(_UpperCAmelCase					)
  __lowerCAmelCase		:    List[str]			     =  len(_UpperCAmelCase					)
  __lowerCAmelCase		:    Any			     =  length >> 1
  return (
      (sorted_list[mid_index] + sorted_list[mid_index - 1]) / 2
      if length % 2 == 0
      else sorted_list[mid_index]
  )




def       A							(      )  ->				Union[str, Any]:


  '''simple docstring'''




  import doctest

  doctest.testmod()


if __name__ == "__main__":
    main()





