import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class SCREAMING_SNAKE_CASE_						(	A__   ,			unittest.TestCase						):
   __lowerCAmelCase      =							MobileBertTokenizer
   __lowerCAmelCase      =							MobileBertTokenizerFast
   __lowerCAmelCase      =							True
   __lowerCAmelCase      =							True
   __lowerCAmelCase      =							filter_non_english
   __lowerCAmelCase      =							"""google/mobilebert-uncased"""







   def 		lowerCamelCase_     (			self		:		Any							):





    """simple docstring"""


    super().setUp()

    UpperCamelCase          =	[
        '''[UNK]''',
        '''[CLS]''',
        '''[SEP]''',
        '''[PAD]''',
        '''[MASK]''',
        '''want''',
        '''##want''',
        '''##ed''',
        '''wa''',
        '''un''',
        '''runn''',
        '''##ing''',
        ''',''',
        '''low''',
        '''lowest''',
    ]
    UpperCamelCase          =	os.path.join(self.tmpdirname					,					VOCAB_FILES_NAMES["""vocab_file"""]							)
    with open(self.vocab_file					,					"""w"""					,					encoding="""utf-8"""							) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

    UpperCamelCase          =	[
        (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
        for tokenizer_def in self.tokenizers_list
    ]







   def 		lowerCamelCase_     (			self		:		Optional[Any]					,					lowerCamelCase_		:		Union[str, Any]							):





    """simple docstring"""


    UpperCamelCase          =	'''UNwant\u00E9d,running'''
    UpperCamelCase          =	'''unwanted, running'''
    return input_text, output_text







   def 		lowerCamelCase_     (			self		:		Optional[Any]							):





    """simple docstring"""


    UpperCamelCase          =	self.tokenizer_class(self.vocab_file							)

    UpperCamelCase          =	tokenizer.tokenize("""UNwant\u00E9d,running"""							)
    self.assertListEqual(__snake_case					,					["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]							)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case							)					,					[9, 6, 7, 12, 10, 11]							)







   def 		lowerCamelCase_     (			self		:		Any							):





    """simple docstring"""


    if not self.test_rust_tokenizer:
     return

    UpperCamelCase          =	self.get_tokenizer()
    UpperCamelCase          =	self.get_rust_tokenizer()

    UpperCamelCase          =	'''UNwant\u00E9d,running'''

    UpperCamelCase          =	tokenizer.tokenize(__snake_case							)
    UpperCamelCase          =	rust_tokenizer.tokenize(__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)

    UpperCamelCase          =	tokenizer.encode(__snake_case					,					add_special_tokens=__snake_case							)
    UpperCamelCase          =	rust_tokenizer.encode(__snake_case					,					add_special_tokens=__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)

    UpperCamelCase          =	self.get_rust_tokenizer()
    UpperCamelCase          =	tokenizer.encode(__snake_case							)
    UpperCamelCase          =	rust_tokenizer.encode(__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)

    # With lower casing
    UpperCamelCase          =	self.get_tokenizer(do_lower_case=__snake_case							)
    UpperCamelCase          =	self.get_rust_tokenizer(do_lower_case=__snake_case							)

    UpperCamelCase          =	'''UNwant\u00E9d,running'''

    UpperCamelCase          =	tokenizer.tokenize(__snake_case							)
    UpperCamelCase          =	rust_tokenizer.tokenize(__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)

    UpperCamelCase          =	tokenizer.encode(__snake_case					,					add_special_tokens=__snake_case							)
    UpperCamelCase          =	rust_tokenizer.encode(__snake_case					,					add_special_tokens=__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)

    UpperCamelCase          =	self.get_rust_tokenizer()
    UpperCamelCase          =	tokenizer.encode(__snake_case							)
    UpperCamelCase          =	rust_tokenizer.encode(__snake_case							)
    self.assertListEqual(__snake_case					,					__snake_case							)







   def 		lowerCamelCase_     (			self		:		List[str]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer()

    self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""							)					,					["""ah""", """\u535A""", """\u63A8""", """zz"""]							)







   def 		lowerCamelCase_     (			self		:		Optional[int]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)					,					["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
    self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)					,					["""hello"""]							)







   def 		lowerCamelCase_     (			self		:		Optional[int]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case					,					strip_accents=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)					,					["""hällo""", """!""", """how""", """are""", """you""", """?"""]							)
    self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)					,					["""h\u00E9llo"""]							)







   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case					,					strip_accents=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)					,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
    self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)					,					["""hello"""]							)







   def 		lowerCamelCase_     (			self		:		str							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)					,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]							)
    self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""							)					,					["""hello"""]							)







   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """							)					,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)







   def 		lowerCamelCase_     (			self		:		Tuple							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case					,					strip_accents=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)					,					["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)







   def 		lowerCamelCase_     (			self		:		Optional[int]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case					,					strip_accents=__snake_case							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """							)					,					["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)







   def 		lowerCamelCase_     (			self		:		Optional[Any]							):





    """simple docstring"""


    UpperCamelCase          =	BasicTokenizer(do_lower_case=__snake_case					,					never_split=["""[UNK]"""]							)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""							)					,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]							)







   def 		lowerCamelCase_     (			self		:		List[Any]							):





    """simple docstring"""


    UpperCamelCase          =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

    UpperCamelCase          =	{}
    for i, token in enumerate(__snake_case							):
     UpperCamelCase          =	i
    UpperCamelCase          =	WordpieceTokenizer(vocab=__snake_case					,					unk_token="""[UNK]"""							)

    self.assertListEqual(tokenizer.tokenize(""""""							)					,					[]							)

    self.assertListEqual(tokenizer.tokenize("""unwanted running"""							)					,					["""un""", """##want""", """##ed""", """runn""", """##ing"""]							)

    self.assertListEqual(tokenizer.tokenize("""unwantedX running"""							)					,					["""[UNK]""", """runn""", """##ing"""]							)







   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    self.assertTrue(_is_whitespace(""" """							)							)
    self.assertTrue(_is_whitespace("""\t"""							)							)
    self.assertTrue(_is_whitespace("""\r"""							)							)
    self.assertTrue(_is_whitespace("""\n"""							)							)
    self.assertTrue(_is_whitespace("""\u00A0"""							)							)

    self.assertFalse(_is_whitespace("""A"""							)							)
    self.assertFalse(_is_whitespace("""-"""							)							)







   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    self.assertTrue(_is_control("""\u0005"""							)							)

    self.assertFalse(_is_control("""A"""							)							)
    self.assertFalse(_is_control(""" """							)							)
    self.assertFalse(_is_control("""\t"""							)							)
    self.assertFalse(_is_control("""\r"""							)							)







   def 		lowerCamelCase_     (			self		:		Optional[int]							):





    """simple docstring"""


    self.assertTrue(_is_punctuation("""-"""							)							)
    self.assertTrue(_is_punctuation("""$"""							)							)
    self.assertTrue(_is_punctuation("""`"""							)							)
    self.assertTrue(_is_punctuation("""."""							)							)

    self.assertFalse(_is_punctuation("""A"""							)							)
    self.assertFalse(_is_punctuation(""" """							)							)







   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    UpperCamelCase          =	self.get_tokenizer()
    UpperCamelCase          =	self.get_rust_tokenizer()

    # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
    self.assertListEqual([tokenizer.tokenize(__snake_case							) for t in ["""Test""", """\xad""", """test"""]]					,					[["""[UNK]"""], [], ["""[UNK]"""]]							)

    self.assertListEqual(
        [rust_tokenizer.tokenize(__snake_case							) for t in ["""Test""", """\xad""", """test"""]]					,					[["""[UNK]"""], [], ["""[UNK]"""]]							)







   @slow
   def 		lowerCamelCase_     (			self		:		Union[str, Any]							):





    """simple docstring"""


    UpperCamelCase          =	self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""							)

    UpperCamelCase          =	tokenizer.encode("""sequence builders"""					,					add_special_tokens=__snake_case							)
    UpperCamelCase          =	tokenizer.encode("""multi-sequence build"""					,					add_special_tokens=__snake_case							)

    UpperCamelCase          =	tokenizer.build_inputs_with_special_tokens(__snake_case							)
    UpperCamelCase          =	tokenizer.build_inputs_with_special_tokens(__snake_case					,					__snake_case							)

    assert encoded_sentence == [101] + text + [102]
    assert encoded_pair == [101] + text + [102] + text_a + [102]







   def 		lowerCamelCase_     (			self		:		Optional[Any]							):





    """simple docstring"""


    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
      UpperCamelCase          =	self.rust_tokenizer_class.from_pretrained(__snake_case					,					**__snake_case							)

      UpperCamelCase          =	f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
      UpperCamelCase          =	tokenizer_r.encode_plus(
          __snake_case					,					return_attention_mask=__snake_case					,					return_token_type_ids=__snake_case					,					return_offsets_mapping=__snake_case					,					add_special_tokens=__snake_case					,					)

      UpperCamelCase          =	tokenizer_r.do_lower_case if hasattr(__snake_case					,					"""do_lower_case"""							) else False
      UpperCamelCase          =	(
          [
              ((0, 0), tokenizer_r.cls_token),
              ((0, 1), '''A'''),
              ((1, 2), ''','''),
              ((3, 5), '''na'''),
              ((5, 6), '''##ï'''),
              ((6, 8), '''##ve'''),
              ((9, 15), tokenizer_r.mask_token),
              ((16, 21), '''Allen'''),
              ((21, 23), '''##NL'''),
              ((23, 24), '''##P'''),
              ((25, 33), '''sentence'''),
              ((33, 34), '''.'''),
              ((0, 0), tokenizer_r.sep_token),
          ]
          if not do_lower_case
          else [
              ((0, 0), tokenizer_r.cls_token),
              ((0, 1), '''a'''),
              ((1, 2), ''','''),
              ((3, 8), '''naive'''),
              ((9, 15), tokenizer_r.mask_token),
              ((16, 21), '''allen'''),
              ((21, 23), '''##nl'''),
              ((23, 24), '''##p'''),
              ((25, 33), '''sentence'''),
              ((33, 34), '''.'''),
              ((0, 0), tokenizer_r.sep_token),
          ]
      )

      self.assertEqual(
          [e[1] for e in expected_results]					,					tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]							)							)
      self.assertEqual([e[0] for e in expected_results]					,					tokens["""offset_mapping"""]							)







   def 		lowerCamelCase_     (			self		:		Optional[int]							):





    """simple docstring"""


    UpperCamelCase          =	['''的''', '''人''', '''有''']
    UpperCamelCase          =	''''''.join(__snake_case							)
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
      UpperCamelCase          =	True
      UpperCamelCase          =	self.tokenizer_class.from_pretrained(__snake_case					,					**__snake_case							)
      UpperCamelCase          =	self.rust_tokenizer_class.from_pretrained(__snake_case					,					**__snake_case							)

      UpperCamelCase          =	tokenizer_p.encode(__snake_case					,					add_special_tokens=__snake_case							)
      UpperCamelCase          =	tokenizer_r.encode(__snake_case					,					add_special_tokens=__snake_case							)

      UpperCamelCase          =	tokenizer_r.convert_ids_to_tokens(__snake_case							)
      UpperCamelCase          =	tokenizer_p.convert_ids_to_tokens(__snake_case							)

      # it is expected that each Chinese character is not preceded by "##"
      self.assertListEqual(__snake_case					,					__snake_case							)
      self.assertListEqual(__snake_case					,					__snake_case							)

      UpperCamelCase          =	False
      UpperCamelCase          =	self.rust_tokenizer_class.from_pretrained(__snake_case					,					**__snake_case							)
      UpperCamelCase          =	self.tokenizer_class.from_pretrained(__snake_case					,					**__snake_case							)

      UpperCamelCase          =	tokenizer_r.encode(__snake_case					,					add_special_tokens=__snake_case							)
      UpperCamelCase          =	tokenizer_p.encode(__snake_case					,					add_special_tokens=__snake_case							)

      UpperCamelCase          =	tokenizer_r.convert_ids_to_tokens(__snake_case							)
      UpperCamelCase          =	tokenizer_p.convert_ids_to_tokens(__snake_case							)

      # it is expected that only the first Chinese character is not preceded by "##".
      UpperCamelCase          =	[
          f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__snake_case							)
      ]
      self.assertListEqual(__snake_case					,					__snake_case							)
      self.assertListEqual(__snake_case					,					__snake_case							)





'''simple docstring'''






# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCamelCase__:   Optional[int]								=			"3"

print("Python version:", sys.version)
print("transformers version:", transformers.__version__)

try:
						import torch

						print("Torch version:", torch.__version__)
						print("Cuda available:", torch.cuda.is_available())
						print("Cuda version:", torch.version.cuda)
						print("CuDNN version:", torch.backends.cudnn.version())
						print("Number of GPUs available:", torch.cuda.device_count())
						print("NCCL version:", torch.cuda.nccl.version())
except ImportError:
						print("Torch version:", None)

try:
						import deepspeed

						print("DeepSpeed version:", deepspeed.__version__)
except ImportError:
						print("DeepSpeed version:", None)

try:
						import tensorflow as tf

						print("TensorFlow version:", tf.__version__)
						print("TF GPUs available:", bool(tf.config.list_physical_devices("GPU")))
						print("Number of TF GPUs available:", len(tf.config.list_physical_devices("GPU")))
except ImportError:
						print("TensorFlow version:", None)





