
import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class     a_       (			a__				,    unittest.TestCase							):



       """simple docstring"""
       __SCREAMING_SNAKE_CASE       :					Dict						=  RealmTokenizer
       __SCREAMING_SNAKE_CASE       :					Optional[Any]						=  RealmTokenizerFast
       __SCREAMING_SNAKE_CASE       :					Any						=  True
       __SCREAMING_SNAKE_CASE       :					Union[str, Any]						=  True
       __SCREAMING_SNAKE_CASE       :					List[str]						=  filter_non_english






       def 	__lowerCAmelCase     (		self   )					->Optional[int]:
           super().setUp()

           SCREAMING_SNAKE_CASE				:				Optional[int]				      = [
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           SCREAMING_SNAKE_CASE				:				Any				      = os.path.join(self.tmpdirname				,       VOCAB_FILES_NAMES['''vocab_file''']   )
           with open(self.vocab_file				,       '''w'''				,       encoding='''utf-8'''   ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->str:
           SCREAMING_SNAKE_CASE				:				int				      = '''UNwant\u00E9d,running'''
           SCREAMING_SNAKE_CASE				:				Any				      = '''unwanted, running'''
           return input_text, output_text






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self.tokenizer_class(self.vocab_file   )

           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = tokenizer.tokenize('''UNwant\u00E9d,running'''   )
           self.assertListEqual(_lowerCamelCase				,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase   )				,       [9, 6, 7, 12, 10, 11]   )






       def 	__lowerCAmelCase     (		self   )					->int:
           if not self.test_rust_tokenizer:
               return

           SCREAMING_SNAKE_CASE				:				Any				      = self.get_tokenizer()
           SCREAMING_SNAKE_CASE				:				int				      = self.get_rust_tokenizer()

           SCREAMING_SNAKE_CASE				:				Optional[int]				      = '''UNwant\u00E9d,running'''

           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = tokenizer.tokenize(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = rust_tokenizer.tokenize(_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = tokenizer.encode(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = rust_tokenizer.encode(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				Dict				      = self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE				:				Any				      = tokenizer.encode(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = rust_tokenizer.encode(_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )

           # With lower casing
           SCREAMING_SNAKE_CASE				:				List[str]				      = self.get_tokenizer(do_lower_case=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = self.get_rust_tokenizer(do_lower_case=_lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				int				      = '''UNwant\u00E9d,running'''

           SCREAMING_SNAKE_CASE				:				Tuple				      = tokenizer.tokenize(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = rust_tokenizer.tokenize(_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				Any				      = tokenizer.encode(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = rust_tokenizer.encode(_lowerCamelCase				,       add_special_tokens=_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				List[Any]				      = self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE				:				Any				      = tokenizer.encode(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				str				      = rust_tokenizer.encode(_lowerCamelCase   )
           self.assertListEqual(_lowerCamelCase				,       _lowerCamelCase   )






       def 	__lowerCAmelCase     (		self   )					->List[Any]:
           SCREAMING_SNAKE_CASE				:				Tuple				      = BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )				,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )






       def 	__lowerCAmelCase     (		self   )					->Optional[int]:
           SCREAMING_SNAKE_CASE				:				Tuple				      = BasicTokenizer(do_lower_case=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,       ['''hello''']   )






       def 	__lowerCAmelCase     (		self   )					->List[str]:
           SCREAMING_SNAKE_CASE				:				Tuple				      = BasicTokenizer(do_lower_case=_lowerCamelCase				,       strip_accents=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,       ['''h\u00E9llo''']   )






       def 	__lowerCAmelCase     (		self   )					->Any:
           SCREAMING_SNAKE_CASE				:				List[str]				      = BasicTokenizer(do_lower_case=_lowerCamelCase				,       strip_accents=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,       ['''hello''']   )






       def 	__lowerCAmelCase     (		self   )					->Union[str, Any]:
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = BasicTokenizer(do_lower_case=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )				,       ['''hello''']   )






       def 	__lowerCAmelCase     (		self   )					->Any:
           SCREAMING_SNAKE_CASE				:				List[str]				      = BasicTokenizer(do_lower_case=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )				,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






       def 	__lowerCAmelCase     (		self   )					->Tuple:
           SCREAMING_SNAKE_CASE				:				int				      = BasicTokenizer(do_lower_case=_lowerCamelCase				,       strip_accents=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				int				      = BasicTokenizer(do_lower_case=_lowerCamelCase				,       strip_accents=_lowerCamelCase   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )				,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






       def 	__lowerCAmelCase     (		self   )					->List[Any]:
           SCREAMING_SNAKE_CASE				:				Dict				      = BasicTokenizer(do_lower_case=_lowerCamelCase				,       never_split=['''[UNK]''']   )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )				,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )






       def 	__lowerCAmelCase     (		self   )					->int:
           SCREAMING_SNAKE_CASE				:				Any				      = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           SCREAMING_SNAKE_CASE				:				int				      = {}
           for i, token in enumerate(_lowerCamelCase   ):
               SCREAMING_SNAKE_CASE				:				Any				      = i
           SCREAMING_SNAKE_CASE				:				str				      = WordpieceTokenizer(vocab=_lowerCamelCase				,       unk_token='''[UNK]'''   )

           self.assertListEqual(tokenizer.tokenize(''''''   )				,       []   )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )				,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )				,       ['''[UNK]''', '''runn''', '''##ing''']   )






       def 	__lowerCAmelCase     (		self   )					->Dict:
           self.assertTrue(_is_whitespace(''' '''   )   )
           self.assertTrue(_is_whitespace('''\t'''   )   )
           self.assertTrue(_is_whitespace('''\r'''   )   )
           self.assertTrue(_is_whitespace('''\n'''   )   )
           self.assertTrue(_is_whitespace('''\u00A0'''   )   )

           self.assertFalse(_is_whitespace('''A'''   )   )
           self.assertFalse(_is_whitespace('''-'''   )   )






       def 	__lowerCAmelCase     (		self   )					->int:
           self.assertTrue(_is_control('''\u0005'''   )   )

           self.assertFalse(_is_control('''A'''   )   )
           self.assertFalse(_is_control(''' '''   )   )
           self.assertFalse(_is_control('''\t'''   )   )
           self.assertFalse(_is_control('''\r'''   )   )






       def 	__lowerCAmelCase     (		self   )					->Any:
           self.assertTrue(_is_punctuation('''-'''   )   )
           self.assertTrue(_is_punctuation('''$'''   )   )
           self.assertTrue(_is_punctuation('''`'''   )   )
           self.assertTrue(_is_punctuation('''.'''   )   )

           self.assertFalse(_is_punctuation('''A'''   )   )
           self.assertFalse(_is_punctuation(''' '''   )   )






       def 	__lowerCAmelCase     (		self   )					->Any:
           SCREAMING_SNAKE_CASE				:				int				      = self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(_lowerCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,       [['''[UNK]'''], [], ['''[UNK]''']]   )

           if self.test_rust_tokenizer:
               SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(_lowerCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']]				,       [['''[UNK]'''], [], ['''[UNK]''']]   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->List[str]:
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

           SCREAMING_SNAKE_CASE				:				int				      = tokenizer.encode('''sequence builders'''				,       add_special_tokens=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				int				      = tokenizer.encode('''multi-sequence build'''				,       add_special_tokens=_lowerCamelCase   )

           SCREAMING_SNAKE_CASE				:				str				      = tokenizer.build_inputs_with_special_tokens(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = tokenizer.build_inputs_with_special_tokens(_lowerCamelCase				,       _lowerCamelCase   )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]






       def 	__lowerCAmelCase     (		self   )					->Any:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                   SCREAMING_SNAKE_CASE				:				List[Any]				      = self.rust_tokenizer_class.from_pretrained(_lowerCamelCase				,       **_lowerCamelCase   )

                   SCREAMING_SNAKE_CASE				:				Tuple				      = F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                   SCREAMING_SNAKE_CASE				:				Dict				      = tokenizer_r.encode_plus(
                       _lowerCamelCase				,       return_attention_mask=_lowerCamelCase				,       return_token_type_ids=_lowerCamelCase				,       return_offsets_mapping=_lowerCamelCase				,       add_special_tokens=_lowerCamelCase				,       )

                   SCREAMING_SNAKE_CASE				:				Optional[int]				      = tokenizer_r.do_lower_case if hasattr(_lowerCamelCase				,       '''do_lower_case'''   ) else False
                   SCREAMING_SNAKE_CASE				:				str				      = (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]				,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
                   self.assertEqual([e[0] for e in expected_results]				,       tokens['''offset_mapping''']   )






       @slow
       def 	__lowerCAmelCase     (		self   )					->Dict:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                   SCREAMING_SNAKE_CASE				:				Optional[int]				      = self.rust_tokenizer_class.from_pretrained(_lowerCamelCase				,       **_lowerCamelCase   )
                   SCREAMING_SNAKE_CASE				:				int				      = self.tokenizer_class.from_pretrained(_lowerCamelCase				,       **_lowerCamelCase   )
                   SCREAMING_SNAKE_CASE				:				Any				      = [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   SCREAMING_SNAKE_CASE				:				str				      = tokenizer_r.batch_encode_candidates(_lowerCamelCase				,       max_length=10				,       return_tensors='''np'''   )
                   SCREAMING_SNAKE_CASE				:				Any				      = tokenizer_p.batch_encode_candidates(_lowerCamelCase				,       max_length=10				,       return_tensors='''np'''   )

                   SCREAMING_SNAKE_CASE				:				int				      = (2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape				,       _lowerCamelCase   )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape				,       _lowerCamelCase   )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape				,       _lowerCamelCase   )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape				,       _lowerCamelCase   )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape				,       _lowerCamelCase   )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape				,       _lowerCamelCase   )


# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
      import tensorflow as tf






class     a_       (			TensorFormatter[Mapping, 'tf.Tensor', Mapping]							):



       """simple docstring"""






       def __init__(		self				,       _lowerCamelCase=None				,       **_lowerCamelCase   )					->int:
           super().__init__(features=_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Any				      = tf_tensor_kwargs
           import tensorflow as tf  # noqa: import tf at initialization






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           import tensorflow as tf

           if isinstance(_lowerCamelCase				,       _lowerCamelCase   ) and column:
               if all(
                   isinstance(_lowerCamelCase				,       tf.Tensor   ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column   ):
                   return tf.stack(_lowerCamelCase   )
               elif all(
                   isinstance(_lowerCamelCase				,       (tf.Tensor, tf.RaggedTensor)   ) and x.ndim == 1 and x.dtype == column[0].dtype
                   for x in column   ):
                   # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
                   return tf.ragged.stack(_lowerCamelCase   )

           return column






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Union[str, Any]:
           import tensorflow as tf

           if value is None:
               return value

           SCREAMING_SNAKE_CASE				:				int				      = {}

           if isinstance(_lowerCamelCase				,       (np.number, np.ndarray)   ) and np.issubdtype(value.dtype				,       np.integer   ):
               SCREAMING_SNAKE_CASE				:				List[Any]				      = {'''dtype''': tf.intaa}
           elif isinstance(_lowerCamelCase				,       (np.number, np.ndarray)   ) and np.issubdtype(value.dtype				,       np.floating   ):
               SCREAMING_SNAKE_CASE				:				Optional[int]				      = {'''dtype''': tf.floataa}
           elif config.PIL_AVAILABLE and "PIL" in sys.modules:
               import PIL.Image

               if isinstance(_lowerCamelCase				,       PIL.Image.Image   ):
                   SCREAMING_SNAKE_CASE				:				Any				      = np.asarray(_lowerCamelCase   )

           return tf.convert_to_tensor(_lowerCamelCase				,       **{**default_dtype, **self.tf_tensor_kwargs}   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Tuple:
           import tensorflow as tf

           # support for torch, tf, jax etc.
           if config.TORCH_AVAILABLE and "torch" in sys.modules:
               import torch

               if isinstance(_lowerCamelCase				,       torch.Tensor   ):
                   return self._tensorize(data_struct.detach().cpu().numpy()[()]   )
           if hasattr(_lowerCamelCase				,       '''__array__'''   ) and not isinstance(_lowerCamelCase				,       tf.Tensor   ):
               SCREAMING_SNAKE_CASE				:				int				      = data_struct.__array__()
           # support for nested types like struct of list of struct
           if isinstance(_lowerCamelCase				,       np.ndarray   ):
               if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
                   return self._consolidate([self.recursive_tensorize(_lowerCamelCase   ) for substruct in data_struct]   )
           elif isinstance(_lowerCamelCase				,       (list, tuple)   ):
               return self._consolidate([self.recursive_tensorize(_lowerCamelCase   ) for substruct in data_struct]   )
           return self._tensorize(_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Optional[int]:
           return map_nested(self._recursive_tensorize				,       _lowerCamelCase				,       map_list=_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Mapping:
           SCREAMING_SNAKE_CASE				:				str				      = self.numpy_arrow_extractor().extract_row(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				List[str]				      = self.python_features_decoder.decode_row(_lowerCamelCase   )
           return self.recursive_tensorize(_lowerCamelCase   )






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->"tf.Tensor":
           SCREAMING_SNAKE_CASE				:				Optional[int]				      = self.numpy_arrow_extractor().extract_column(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				str				      = self.python_features_decoder.decode_column(_lowerCamelCase				,       pa_table.column_names[0]   )
           SCREAMING_SNAKE_CASE				:				str				      = self.recursive_tensorize(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				int				      = self._consolidate(_lowerCamelCase   )
           return column






       def 	__lowerCAmelCase     (		self				,       _lowerCamelCase   )					->Mapping:
           SCREAMING_SNAKE_CASE				:				Tuple				      = self.numpy_arrow_extractor().extract_batch(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				Dict				      = self.python_features_decoder.decode_batch(_lowerCamelCase   )
           SCREAMING_SNAKE_CASE				:				int				      = self.recursive_tensorize(_lowerCamelCase   )
           for column_name in batch:
               SCREAMING_SNAKE_CASE				:				Optional[Any]				      = self._consolidate(batch[column_name]   )
           return batch
