

'''simple docstring'''



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 						lowerCAmelCase__   (      A__ ,					unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      MobileBertTokenizer
 lowerCAmelCase__  					=      MobileBertTokenizerFast
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      filter_non_english
 lowerCAmelCase__  					=      '''google/mobilebert-uncased'''
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Tuple:


      """simple docstring"""




      super().setUp()

      __SCREAMING_SNAKE_CASE     =      [
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      __SCREAMING_SNAKE_CASE     =      os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES["""vocab_file"""]			)
      with open(self.vocab_file					,							"""w"""					,							encoding="""utf-8"""			) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)

      __SCREAMING_SNAKE_CASE     =      [
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]
 def       UpperCAmelCase__		(		self					:       int					,							__SCREAMING_SNAKE_CASE					:       List[str]			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      '''UNwant\u00E9d,running'''
      __SCREAMING_SNAKE_CASE     =      '''unwanted, running'''
      return input_text, output_text
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer_class(self.vocab_file			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize("""UNwant\u00E9d,running"""			)
      self.assertListEqual(lowerCAmelCase__					,							["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)					,							[9, 6, 7, 12, 10, 11]			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Tuple:


      """simple docstring"""




      if not self.test_rust_tokenizer:
           return

      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()
      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer()

      __SCREAMING_SNAKE_CASE     =      '''UNwant\u00E9d,running'''

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize(lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.tokenize(lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer()
      __SCREAMING_SNAKE_CASE     =      tokenizer.encode(lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.encode(lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

      # With lower casing
      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer(do_lower_case=lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer(do_lower_case=lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      '''UNwant\u00E9d,running'''

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize(lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.tokenize(lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer()
      __SCREAMING_SNAKE_CASE     =      tokenizer.encode(lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      rust_tokenizer.encode(lowerCAmelCase__			)
      self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			)					,							["""ah""", """\u535A""", """\u63A8""", """zz"""]			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)					,							["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)					,							["""hello"""]			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__					,							strip_accents=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)					,							["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)					,							["""h\u00E9llo"""]			)
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__					,							strip_accents=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)					,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)					,							["""hello"""]			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)					,							["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
      self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)					,							["""hello"""]			)
 def       UpperCAmelCase__		(		self					:       Any			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)					,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)
 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__					,							strip_accents=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)					,							["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__					,							strip_accents=lowerCAmelCase__			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)					,							["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      BasicTokenizer(do_lower_case=lowerCAmelCase__					,							never_split=["""[UNK]"""]			)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			)					,							["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)
 def       UpperCAmelCase__		(		self					:       int			)  -> List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      __SCREAMING_SNAKE_CASE     =      {}
      for i, token in enumerate(lowerCAmelCase__			):
           __SCREAMING_SNAKE_CASE     =      i
      __SCREAMING_SNAKE_CASE     =      WordpieceTokenizer(vocab=lowerCAmelCase__					,							unk_token="""[UNK]"""			)

      self.assertListEqual(tokenizer.tokenize(""""""			)					,							[]			)

      self.assertListEqual(tokenizer.tokenize("""unwanted running"""			)					,							["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

      self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			)					,							["""[UNK]""", """runn""", """##ing"""]			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> List[Any]:


      """simple docstring"""




      self.assertTrue(_is_whitespace(""" """			)			)
      self.assertTrue(_is_whitespace("""\t"""			)			)
      self.assertTrue(_is_whitespace("""\r"""			)			)
      self.assertTrue(_is_whitespace("""\n"""			)			)
      self.assertTrue(_is_whitespace("""\u00A0"""			)			)

      self.assertFalse(_is_whitespace("""A"""			)			)
      self.assertFalse(_is_whitespace("""-"""			)			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[int]:


      """simple docstring"""




      self.assertTrue(_is_control("""\u0005"""			)			)

      self.assertFalse(_is_control("""A"""			)			)
      self.assertFalse(_is_control(""" """			)			)
      self.assertFalse(_is_control("""\t"""			)			)
      self.assertFalse(_is_control("""\r"""			)			)
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Dict:


      """simple docstring"""




      self.assertTrue(_is_punctuation("""-"""			)			)
      self.assertTrue(_is_punctuation("""$"""			)			)
      self.assertTrue(_is_punctuation("""`"""			)			)
      self.assertTrue(_is_punctuation("""."""			)			)

      self.assertFalse(_is_punctuation("""A"""			)			)
      self.assertFalse(_is_punctuation(""" """			)			)
 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.get_tokenizer()
      __SCREAMING_SNAKE_CASE     =      self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(lowerCAmelCase__			) for t in ["""Test""", """\xad""", """test"""]]					,							[["""[UNK]"""], [], ["""[UNK]"""]]			)

      self.assertListEqual(
          [rust_tokenizer.tokenize(lowerCAmelCase__			) for t in ["""Test""", """\xad""", """test"""]]					,							[["""[UNK]"""], [], ["""[UNK]"""]]			)
 @slow
 def       UpperCAmelCase__		(		self					:       Tuple			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.encode("""sequence builders"""					,							add_special_tokens=lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      tokenizer.encode("""multi-sequence build"""					,							add_special_tokens=lowerCAmelCase__			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__			)
      __SCREAMING_SNAKE_CASE     =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__					,							lowerCAmelCase__			)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> int:


      """simple docstring"""




      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__					,							**lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode_plus(
                    lowerCAmelCase__					,							return_attention_mask=lowerCAmelCase__					,							return_token_type_ids=lowerCAmelCase__					,							return_offsets_mapping=lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__					,							)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.do_lower_case if hasattr(lowerCAmelCase__					,							"""do_lower_case"""			) else False
                __SCREAMING_SNAKE_CASE     =      (
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''A'''),
                        ((1, 2), ''','''),
                        ((3, 5), '''na'''),
                        ((5, 6), '''##ï'''),
                        ((6, 8), '''##ve'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''Allen'''),
                        ((21, 23), '''##NL'''),
                        ((23, 24), '''##P'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''a'''),
                        ((1, 2), ''','''),
                        ((3, 8), '''naive'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''allen'''),
                        ((21, 23), '''##nl'''),
                        ((23, 24), '''##p'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]					,							tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]			)			)
                self.assertEqual([e[0] for e in expected_results]					,							tokens["""offset_mapping"""]			)


 def       UpperCAmelCase__		(		self					:       Dict			)  -> List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      ['''的''', '''人''', '''有''']
      __SCREAMING_SNAKE_CASE     =      ''''''.join(lowerCAmelCase__			)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      True
                __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(lowerCAmelCase__					,							**lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__					,							**lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_p.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__			)

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)
                self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      False
                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__					,							**lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(lowerCAmelCase__					,							**lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.encode(lowerCAmelCase__					,							add_special_tokens=lowerCAmelCase__			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__			)

                # it is expected that only the first Chinese character is not preceded by "##".
                __SCREAMING_SNAKE_CASE     =      [
                    f'##{token}' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase__			)
                ]
                self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)
                self.assertListEqual(lowerCAmelCase__					,							lowerCAmelCase__			)










from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:NestedDataStructureLike[PathLike]			,			lowerCAmelCase__			:Optional[NamedSplit] = None			,			lowerCAmelCase__			:Optional[Features] = None			,			lowerCAmelCase__			:str = None			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:Optional[int] = None			,			**lowerCAmelCase__			:Optional[int]			,			)		->			Tuple:
										super().__init__(
										    lowerCAmelCase__			,			split=lowerCAmelCase__			,			features=lowerCAmelCase__			,			cache_dir=lowerCAmelCase__			,			keep_in_memory=lowerCAmelCase__			,			streaming=lowerCAmelCase__			,			num_proc=lowerCAmelCase__			,			**lowerCAmelCase__			,			)
										__SCREAMING_SNAKE_CASE						:     List[str]					=		path_or_paths if isinstance(lowerCAmelCase__			,			lowerCAmelCase__	) else {self.split: path_or_paths}
										__SCREAMING_SNAKE_CASE						:     int					=		Text(
										    cache_dir=lowerCAmelCase__			,			data_files=lowerCAmelCase__			,			features=lowerCAmelCase__			,			**lowerCAmelCase__			,			)



				def 		__magic_name__(       self			:Dict	)		->			Tuple:
										# Build iterable dataset
										if self.streaming:
																__SCREAMING_SNAKE_CASE						:     int					=		self.builder.as_streaming_dataset(split=self.split	)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		None
																__SCREAMING_SNAKE_CASE						:     str					=		None
																__SCREAMING_SNAKE_CASE						:     Dict					=		None
																__SCREAMING_SNAKE_CASE						:     Tuple					=		None

																self.builder.download_and_prepare(
																    download_config=lowerCAmelCase__			,			download_mode=lowerCAmelCase__			,			verification_mode=lowerCAmelCase__			,			base_path=lowerCAmelCase__			,			num_proc=self.num_proc			,			)
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.builder.as_dataset(
																    split=self.split			,			verification_mode=lowerCAmelCase__			,			in_memory=self.keep_in_memory	)
										return dataset
