
"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     A_	(     _lowerCamelCase      ,					unittest.TestCase   ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_	      =   RoCBertTokenizer
							SCREAMING_SNAKE_CASE_	      =   None
							SCREAMING_SNAKE_CASE_	      =   False
							SCREAMING_SNAKE_CASE_	      =   True
							SCREAMING_SNAKE_CASE_	      =   filter_non_english





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										super().setUp()

										lowerCamelCase__	:     Union[str, Any]              =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
										lowerCamelCase__	:     Dict              ={}
										lowerCamelCase__	:     Tuple              ={}
										for i, value in enumerate(lowercase_       ):
													lowerCamelCase__	:     Tuple              =i
													lowerCamelCase__	:     Dict              =i
										lowerCamelCase__	:     Union[str, Any]              =os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['vocab_file']       )
										lowerCamelCase__	:     Union[str, Any]              =os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['word_shape_file']       )
										lowerCamelCase__	:     List[Any]              =os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['word_pronunciation_file']       )
										with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
										with open(self.word_shape_file     ,							'w'     ,							encoding='utf-8'       ) as word_shape_writer:
													json.dump(lowercase_     ,							lowercase_     ,							ensure_ascii=lowercase_       )
										with open(self.word_pronunciation_file     ,							'w'     ,							encoding='utf-8'       ) as word_pronunciation_writer:
													json.dump(lowercase_     ,							lowercase_     ,							ensure_ascii=lowercase_       )





							def       UpperCAmelCase__							(  self  :List[str]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Optional[Any]              =self.tokenizer_class(self.vocab_file     ,							self.word_shape_file     ,							self.word_pronunciation_file       )

										lowerCamelCase__	:     List[str]              =tokenizer.tokenize('你好[SEP]你是谁'       )
										self.assertListEqual(lowercase_     ,							['你', '好', '[SEP]', '你', '是', '谁']       )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_       )     ,							[5, 6, 2, 5, 7, 8]       )
										self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowercase_       )     ,							[5, 6, 2, 5, 7, 8]       )
										self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowercase_       )     ,							[5, 6, 2, 5, 7, 8]       )





							def       UpperCAmelCase__							(  self  :str       ):





										"""simple docstring"""

										lowerCamelCase__	:     Tuple              =RoCBertBasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'       )     ,							['ah', '\u535A', '\u63A8', 'zz']       )





							def       UpperCAmelCase__							(  self  :Tuple       ):





										"""simple docstring"""

										lowerCamelCase__	:     Any              =RoCBertBasicTokenizer(do_lower_case=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )     ,							['hello', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )     ,							['hello']       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Union[str, Any]              =RoCBertBasicTokenizer(do_lower_case=lowercase_     ,							strip_accents=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )     ,							['hällo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )     ,							['h\u00E9llo']       )





							def       UpperCAmelCase__							(  self  :str       ):





										"""simple docstring"""

										lowerCamelCase__	:     Any              =RoCBertBasicTokenizer(do_lower_case=lowercase_     ,							strip_accents=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )     ,							['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )     ,							['hello']       )





							def       UpperCAmelCase__							(  self  :List[str]       ):





										"""simple docstring"""

										lowerCamelCase__	:     str              =RoCBertBasicTokenizer(do_lower_case=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )     ,							['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )     ,							['hello']       )





							def       UpperCAmelCase__							(  self  :int       ):





										"""simple docstring"""

										lowerCamelCase__	:     Optional[Any]              =RoCBertBasicTokenizer(do_lower_case=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )     ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?']       )





							def       UpperCAmelCase__							(  self  :List[str]       ):





										"""simple docstring"""

										lowerCamelCase__	:     int              =RoCBertBasicTokenizer(do_lower_case=lowercase_     ,							strip_accents=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )     ,							['HäLLo', '!', 'how', 'Are', 'yoU', '?']       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Tuple              =RoCBertBasicTokenizer(do_lower_case=lowercase_     ,							strip_accents=lowercase_       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )     ,							['HaLLo', '!', 'how', 'Are', 'yoU', '?']       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Union[str, Any]              =RoCBertBasicTokenizer(do_lower_case=lowercase_     ,							never_split=['[UNK]']       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'       )     ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Dict              =['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										lowerCamelCase__	:     int              ={}
										for i, token in enumerate(lowercase_       ):
													lowerCamelCase__	:     Optional[Any]              =i
										lowerCamelCase__	:     Optional[Any]              =RoCBertWordpieceTokenizer(vocab=lowercase_     ,							unk_token='[UNK]'       )

										self.assertListEqual(tokenizer.tokenize(''       )     ,							[]       )

										self.assertListEqual(tokenizer.tokenize('unwanted running'       )     ,							['un', '##want', '##ed', 'runn', '##ing']       )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'       )     ,							['[UNK]', 'runn', '##ing']       )





							def       UpperCAmelCase__							(  self  :Optional[int]       ):





										"""simple docstring"""

										self.assertTrue(_is_whitespace(' '       )       )
										self.assertTrue(_is_whitespace('\t'       )       )
										self.assertTrue(_is_whitespace('\r'       )       )
										self.assertTrue(_is_whitespace('\n'       )       )
										self.assertTrue(_is_whitespace('\u00A0'       )       )

										self.assertFalse(_is_whitespace('A'       )       )
										self.assertFalse(_is_whitespace('-'       )       )





							def       UpperCAmelCase__							(  self  :List[Any]       ):





										"""simple docstring"""

										self.assertTrue(_is_control('\u0005'       )       )

										self.assertFalse(_is_control('A'       )       )
										self.assertFalse(_is_control(' '       )       )
										self.assertFalse(_is_control('\t'       )       )
										self.assertFalse(_is_control('\r'       )       )





							def       UpperCAmelCase__							(  self  :List[str]       ):





										"""simple docstring"""

										self.assertTrue(_is_punctuation('-'       )       )
										self.assertTrue(_is_punctuation('$'       )       )
										self.assertTrue(_is_punctuation('`'       )       )
										self.assertTrue(_is_punctuation('.'       )       )

										self.assertFalse(_is_punctuation('A'       )       )
										self.assertFalse(_is_punctuation(' '       )       )





							def       UpperCAmelCase__							(  self  :str       ):





										"""simple docstring"""

										lowerCamelCase__	:     Optional[int]              =self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowercase_       ) for t in ['Test', '\xad', 'test']]     ,							[['[UNK]'], [], ['[UNK]']]       )

										if self.test_rust_tokenizer:
													lowerCamelCase__	:     Optional[Any]              =self.get_rust_tokenizer()
													self.assertListEqual(
													    [rust_tokenizer.tokenize(lowercase_       ) for t in ['Test', '\xad', 'test']]     ,							[['[UNK]'], [], ['[UNK]']]       )





							def       UpperCAmelCase__							(  self  :Dict       ):





										"""simple docstring"""

										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																lowerCamelCase__	:     str              =self.rust_tokenizer_class.from_pretrained(lowercase_     ,							**lowercase_       )

																lowerCamelCase__	:     Optional[int]              =f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																lowerCamelCase__	:     Optional[Any]              =tokenizer_r.encode_plus(
																    lowercase_     ,							return_attention_mask=lowercase_     ,							return_token_type_ids=lowercase_     ,							return_offsets_mapping=lowercase_     ,							add_special_tokens=lowercase_     ,							)

																lowerCamelCase__	:     Tuple              =tokenizer_r.do_lower_case if hasattr(lowercase_     ,							'do_lower_case'       ) else False
																lowerCamelCase__	:     Tuple              =(
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'A'),
																        ((1, 2), ','),
																        ((3, 5), 'na'),
																        ((5, 6), '##ï'),
																        ((6, 8), '##ve'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'Allen'),
																        ((21, 23), '##NL'),
																        ((23, 24), '##P'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'a'),
																        ((1, 2), ','),
																        ((3, 8), 'naive'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'allen'),
																        ((21, 23), '##nl'),
																        ((23, 24), '##p'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]     ,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']       )       )
																self.assertEqual([e[0] for e in expected_results]     ,							tokens['offset_mapping']       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     List[str]              =['的', '人', '有']
										lowerCamelCase__	:     List[str]              =''.join(lowercase_       )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																lowerCamelCase__	:     str              =True
																lowerCamelCase__	:     Any              =self.tokenizer_class.from_pretrained(lowercase_     ,							**lowercase_       )
																lowerCamelCase__	:     Tuple              =self.rust_tokenizer_class.from_pretrained(lowercase_     ,							**lowercase_       )

																lowerCamelCase__	:     Dict              =tokenizer_p.encode(lowercase_     ,							add_special_tokens=lowercase_       )
																lowerCamelCase__	:     List[str]              =tokenizer_r.encode(lowercase_     ,							add_special_tokens=lowercase_       )

																lowerCamelCase__	:     Any              =tokenizer_r.convert_ids_to_tokens(lowercase_       )
																lowerCamelCase__	:     Union[str, Any]              =tokenizer_p.convert_ids_to_tokens(lowercase_       )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(lowercase_     ,							lowercase_       )
																self.assertListEqual(lowercase_     ,							lowercase_       )

																lowerCamelCase__	:     str              =False
																lowerCamelCase__	:     Dict              =self.rust_tokenizer_class.from_pretrained(lowercase_     ,							**lowercase_       )
																lowerCamelCase__	:     List[str]              =self.tokenizer_class.from_pretrained(lowercase_     ,							**lowercase_       )

																lowerCamelCase__	:     Optional[int]              =tokenizer_r.encode(lowercase_     ,							add_special_tokens=lowercase_       )
																lowerCamelCase__	:     Optional[Any]              =tokenizer_p.encode(lowercase_     ,							add_special_tokens=lowercase_       )

																lowerCamelCase__	:     Tuple              =tokenizer_r.convert_ids_to_tokens(lowercase_       )
																lowerCamelCase__	:     Optional[int]              =tokenizer_p.convert_ids_to_tokens(lowercase_       )

																# it is expected that only the first Chinese character is not preceded by "##".
																lowerCamelCase__	:     int              =[
																    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowercase_       )
																]
																self.assertListEqual(lowercase_     ,							lowercase_       )
																self.assertListEqual(lowercase_     ,							lowercase_       )





							@slow
							def       UpperCAmelCase__							(  self  :Tuple       ):





										"""simple docstring"""

										lowerCamelCase__	:     Union[str, Any]              =self.tokenizer_class(self.vocab_file     ,							self.word_shape_file     ,							self.word_pronunciation_file       )

										lowerCamelCase__	:     Optional[Any]              =tokenizer.encode('你好'     ,							add_special_tokens=lowercase_       )
										lowerCamelCase__	:     Any              =tokenizer.encode('你是谁'     ,							add_special_tokens=lowercase_       )

										lowerCamelCase__	:     Optional[int]              =tokenizer.build_inputs_with_special_tokens(lowercase_       )
										lowerCamelCase__	:     Tuple              =tokenizer.build_inputs_with_special_tokens(lowercase_     ,							lowercase_       )

										assert encoded_sentence == [1] + text + [2]
										assert encoded_pair == [1] + text + [2] + text_a + [2]





							def       UpperCAmelCase__							(  self  :List[Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     int              =self.get_tokenizers(do_lower_case=lowercase_       )
										for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
																lowerCamelCase__	:     List[Any]              ='你好，你是谁'
																lowerCamelCase__	:     Optional[int]              =tokenizer.tokenize(lowercase_       )
																lowerCamelCase__	:     Dict              =tokenizer.convert_tokens_to_ids(lowercase_       )
																lowerCamelCase__	:     List[str]              =tokenizer.convert_tokens_to_shape_ids(lowercase_       )
																lowerCamelCase__	:     str              =tokenizer.convert_tokens_to_pronunciation_ids(lowercase_       )
																lowerCamelCase__	:     List[str]              =tokenizer.prepare_for_model(
																    lowercase_     ,							lowercase_     ,							lowercase_     ,							add_special_tokens=lowercase_       )

																lowerCamelCase__	:     str              =tokenizer.encode_plus(lowercase_     ,							add_special_tokens=lowercase_       )

																self.assertEqual(lowercase_     ,							lowercase_       )





'''simple docstring'''

# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
a  :				int									=  abspath(join(dirname(__file__), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					config.addinivalue_line(
					    '''markers''',      '''is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested'''	)
					config.addinivalue_line(
					    '''markers''',      '''is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested'''	)
					config.addinivalue_line('''markers''',      '''is_pipeline_test: mark test to run only when pipelines are tested'''	)
					config.addinivalue_line('''markers''',      '''is_staging_test: mark test to run only in the staging environment'''	)
					config.addinivalue_line('''markers''',      '''accelerate_tests: mark test that require accelerate'''	)
					config.addinivalue_line('''markers''',      '''tool_tests: mark the tool tests that are run on their specific schedule'''	)


def 		__magic_name__    (	__UpperCAmelCase	)		->					int:

					'''simple docstring'''

					from transformers.testing_utils import pytest_addoption_shared

					pytest_addoption_shared(__UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase	)		->					Union[str, Any]:

					'''simple docstring'''

					from transformers.testing_utils import pytest_terminal_summary_main

					snake_case_									=				terminalreporter.config.getoption('''--make-reports'''	)
					if make_reports:
										pytest_terminal_summary_main(__UpperCAmelCase,      id=__UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					if exitstatus == 5:
										snake_case_									=				0


# Doctest custom flag to ignore output.
a  :				Union[str, Any]									=  doctest.register_optionflag('IGNORE_RESULT')

a  :				Optional[int]									=  doctest.OutputChecker






class 						a				(   _lowerCamelCase     ):


				def   A_ (		self    :					List[Any]       ,				lowercase_    :					int       ,				lowercase_    :					Tuple       ,				lowercase_    :					Optional[int]					):
									if IGNORE_RESULT & optionflags:
														return True
									return OutputChecker.check_output(self       ,				lowercase_       ,				lowercase_       ,				lowercase_					)







a  :				List[Any]									=  CustomOutputChecker
a  :				Optional[int]									=  HfDoctestModule

a  :				Tuple									=  HfDocTestParser

