


'''simple docstring'''




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		UpperCAmelCase_      (  __lowercase       ,	unittest.TestCase		):
     lowerCamelCase				:	List[str]		=     RoCBertTokenizer
     lowerCamelCase				:	Tuple		=     None
     lowerCamelCase				:	int		=     False
     lowerCamelCase				:	Tuple		=     True
     lowerCamelCase				:	int		=     filter_non_english





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Dict:
        super().setUp()

        lowerCAmelCase     =			['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
        lowerCAmelCase     =			{}
        lowerCAmelCase     =			{}
        for i, value in enumerate(UpperCAmelCase__    ):
           lowerCAmelCase     =			i
           lowerCAmelCase     =			i
        lowerCAmelCase     =			os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['vocab_file']    )
        lowerCAmelCase     =			os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['word_shape_file']    )
        lowerCAmelCase     =			os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['word_pronunciation_file']    )
        with open(self.vocab_file    ,      'w'    ,      encoding='utf-8'    ) as vocab_writer:
           vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )
        with open(self.word_shape_file    ,      'w'    ,      encoding='utf-8'    ) as word_shape_writer:
           json.dump(UpperCAmelCase__    ,      UpperCAmelCase__    ,      ensure_ascii=UpperCAmelCase__    )
        with open(self.word_pronunciation_file    ,      'w'    ,      encoding='utf-8'    ) as word_pronunciation_writer:
           json.dump(UpperCAmelCase__    ,      UpperCAmelCase__    ,      ensure_ascii=UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Optional[Any]:
        lowerCAmelCase     =			self.tokenizer_class(self.vocab_file    ,      self.word_shape_file    ,      self.word_pronunciation_file    )

        lowerCAmelCase     =			tokenizer.tokenize('你好[SEP]你是谁'    )
        self.assertListEqual(UpperCAmelCase__    ,      ['你', '好', '[SEP]', '你', '是', '谁']    )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__    )    ,      [5, 6, 2, 5, 7, 8]    )
        self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase__    )    ,      [5, 6, 2, 5, 7, 8]    )
        self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase__    )    ,      [5, 6, 2, 5, 7, 8]    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  str:
        lowerCAmelCase     =			RoCBertBasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )    ,      ['ah', '\u535A', '\u63A8', 'zz']    )





     def 							__UpperCAmelCase      ( self       :      Optional[Any]    )				->  str:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    ,      ['hello', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,      ['hello']    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    )				->  Any:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    ,      strip_accents=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,      ['hällo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,      ['h\u00E9llo']    )





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  List[str]:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    ,      strip_accents=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,      ['hallo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,      ['hello']    )





     def 							__UpperCAmelCase      ( self       :      str    )				->  List[Any]:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,      ['hallo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,      ['hello']    )





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  List[str]:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  str:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    ,      strip_accents=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    )				->  Any:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    ,      strip_accents=UpperCAmelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  List[str]:
        lowerCAmelCase     =			RoCBertBasicTokenizer(do_lower_case=UpperCAmelCase__    ,      never_split=['[UNK]']    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )    ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )





     def 							__UpperCAmelCase      ( self       :      str    )				->  List[str]:
        lowerCAmelCase     =			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

        lowerCAmelCase     =			{}
        for i, token in enumerate(UpperCAmelCase__    ):
           lowerCAmelCase     =			i
        lowerCAmelCase     =			RoCBertWordpieceTokenizer(vocab=UpperCAmelCase__    ,      unk_token='[UNK]'    )

        self.assertListEqual(tokenizer.tokenize(''    )    ,      []    )

        self.assertListEqual(tokenizer.tokenize('unwanted running'    )    ,      ['un', '##want', '##ed', 'runn', '##ing']    )

        self.assertListEqual(tokenizer.tokenize('unwantedX running'    )    ,      ['[UNK]', 'runn', '##ing']    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  Tuple:
        self.assertTrue(_is_whitespace(' '    )    )
        self.assertTrue(_is_whitespace('\t'    )    )
        self.assertTrue(_is_whitespace('\r'    )    )
        self.assertTrue(_is_whitespace('\n'    )    )
        self.assertTrue(_is_whitespace('\u00A0'    )    )

        self.assertFalse(_is_whitespace('A'    )    )
        self.assertFalse(_is_whitespace('-'    )    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    )				->  List[str]:
        self.assertTrue(_is_control('\u0005'    )    )

        self.assertFalse(_is_control('A'    )    )
        self.assertFalse(_is_control(' '    )    )
        self.assertFalse(_is_control('\t'    )    )
        self.assertFalse(_is_control('\r'    )    )





     def 							__UpperCAmelCase      ( self       :      Optional[int]    )				->  List[Any]:
        self.assertTrue(_is_punctuation('-'    )    )
        self.assertTrue(_is_punctuation('$'    )    )
        self.assertTrue(_is_punctuation('`'    )    )
        self.assertTrue(_is_punctuation('.'    )    )

        self.assertFalse(_is_punctuation('A'    )    )
        self.assertFalse(_is_punctuation(' '    )    )





     def 							__UpperCAmelCase      ( self       :      Union[str, Any]    )				->  Optional[int]:
        lowerCAmelCase     =			self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__    ) for t in ['Test', '\xad', 'test']]    ,      [['[UNK]'], [], ['[UNK]']]    )

        if self.test_rust_tokenizer:
           lowerCAmelCase     =			self.get_rust_tokenizer()
           self.assertListEqual(
               [rust_tokenizer.tokenize(UpperCAmelCase__    ) for t in ['Test', '\xad', 'test']]    ,      [['[UNK]'], [], ['[UNK]']]    )





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Tuple:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
              lowerCAmelCase     =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__    ,      **UpperCAmelCase__    )

              lowerCAmelCase     =			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
              lowerCAmelCase     =			tokenizer_r.encode_plus(
                  UpperCAmelCase__    ,      return_attention_mask=UpperCAmelCase__    ,      return_token_type_ids=UpperCAmelCase__    ,      return_offsets_mapping=UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    ,      )

              lowerCAmelCase     =			tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__    ,      'do_lower_case'    ) else False
              lowerCAmelCase     =			(
                  [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'A'),
                      ((1, 2), ','),
                      ((3, 5), 'na'),
                      ((5, 6), '##ï'),
                      ((6, 8), '##ve'),
                      ((9, 1_5), tokenizer_r.mask_token),
                      ((1_6, 2_1), 'Allen'),
                      ((2_1, 2_3), '##NL'),
                      ((2_3, 2_4), '##P'),
                      ((2_5, 3_3), 'sentence'),
                      ((3_3, 3_4), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
                  if not do_lower_case
                  else [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'a'),
                      ((1, 2), ','),
                      ((3, 8), 'naive'),
                      ((9, 1_5), tokenizer_r.mask_token),
                      ((1_6, 2_1), 'allen'),
                      ((2_1, 2_3), '##nl'),
                      ((2_3, 2_4), '##p'),
                      ((2_5, 3_3), 'sentence'),
                      ((3_3, 3_4), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
              )

              self.assertEqual(
                  [e[1] for e in expected_results]    ,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
              self.assertEqual([e[0] for e in expected_results]    ,      tokens['offset_mapping']    )





     def 							__UpperCAmelCase      ( self       :      str    )				->  Tuple:
        lowerCAmelCase     =			['的', '人', '有']
        lowerCAmelCase     =			''.join(UpperCAmelCase__    )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
              lowerCAmelCase     =			True
              lowerCAmelCase     =			self.tokenizer_class.from_pretrained(UpperCAmelCase__    ,      **UpperCAmelCase__    )
              lowerCAmelCase     =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__    ,      **UpperCAmelCase__    )

              lowerCAmelCase     =			tokenizer_p.encode(UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer_r.encode(UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )

              lowerCAmelCase     =			tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__    )

              # it is expected that each Chinese character is not preceded by "##"
              self.assertListEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )
              self.assertListEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

              lowerCAmelCase     =			False
              lowerCAmelCase     =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__    ,      **UpperCAmelCase__    )
              lowerCAmelCase     =			self.tokenizer_class.from_pretrained(UpperCAmelCase__    ,      **UpperCAmelCase__    )

              lowerCAmelCase     =			tokenizer_r.encode(UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer_p.encode(UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )

              lowerCAmelCase     =			tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__    )

              # it is expected that only the first Chinese character is not preceded by "##".
              lowerCAmelCase     =			[
                  F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__    )
              ]
              self.assertListEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )
              self.assertListEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     @slow
     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  Optional[int]:
        lowerCAmelCase     =			self.tokenizer_class(self.vocab_file    ,      self.word_shape_file    ,      self.word_pronunciation_file    )

        lowerCAmelCase     =			tokenizer.encode('你好'    ,      add_special_tokens=UpperCAmelCase__    )
        lowerCAmelCase     =			tokenizer.encode('你是谁'    ,      add_special_tokens=UpperCAmelCase__    )

        lowerCAmelCase     =			tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__    )
        lowerCAmelCase     =			tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__    ,      UpperCAmelCase__    )

        assert encoded_sentence == [1] + text + [2]
        assert encoded_pair == [1] + text + [2] + text_a + [2]





     def 							__UpperCAmelCase      ( self       :      str    )				->  Optional[Any]:
        lowerCAmelCase     =			self.get_tokenizers(do_lower_case=UpperCAmelCase__    )
        for tokenizer in tokenizers:
           with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
              lowerCAmelCase     =			'你好，你是谁'
              lowerCAmelCase     =			tokenizer.tokenize(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer.convert_tokens_to_ids(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer.convert_tokens_to_shape_ids(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCAmelCase__    )
              lowerCAmelCase     =			tokenizer.prepare_for_model(
                  UpperCAmelCase__    ,      UpperCAmelCase__    ,      UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )

              lowerCAmelCase     =			tokenizer.encode_plus(UpperCAmelCase__    ,      add_special_tokens=UpperCAmelCase__    )

              self.assertEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )









'''simple docstring'''




import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


__snake_case    						=logging.getLogger(__name__)

@dataclass
@add_start_docstrings(TrainingArguments.__doc__		)
class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	bool		=     field(default=__lowercase       ,	metadata={'''help''': '''Whether to use SortishSampler or not.'''}		)
     lowerCamelCase				:	bool		=     field(
         default=__lowercase       ,	metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}		)
     lowerCamelCase				:	Optional[int]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': (
                 '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `max_length` value of the model configuration.'''
             )
         }       ,	)
     lowerCamelCase				:	Optional[int]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': (
                 '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `num_beams` value of the model configuration.'''
             )
         }       ,	)
     lowerCamelCase				:	Optional[Union[str, Path, GenerationConfig]]		=     field(
         default=__lowercase       ,	metadata={
             '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
         }       ,	)





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  List[str]:
        lowerCAmelCase     =			super().to_dict()
        for k, v in d.items():
           if isinstance(UpperCAmelCase__    ,      UpperCAmelCase__    ):
              lowerCAmelCase     =			v.to_dict()
        return d





