




"""simple docstring"""

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a					=			get_tests_dir('''fixtures/spiece.model''')





@require_sentencepiece
@require_tokenizers
class 				lowercase_       (     __UpperCamelCase			,	unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase    :     Tuple        =	DebertaVaTokenizer
 UpperCAmelCase    :     int        =	DebertaVaTokenizerFast
 UpperCAmelCase    :     Optional[int]        =	True
 UpperCAmelCase    :     Optional[Any]        =	True

 def 			lowerCAmelCase_							( self       :       Dict		):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							unk_token='<unk>'		)
       tokenizer.save_pretrained(self.tmpdirname		)

 def 			lowerCAmelCase_							( self       :       Dict    ,							_UpperCAmelCase       :       Tuple		):
       _A					   =   'this is a test'
       _A					   =   'this is a test'
       return input_text, output_text

 def 			lowerCAmelCase_							( self       :       Union[str, Any]		):
       _A					   =   '<pad>'
       _A					   =   0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase		)    ,							_UpperCAmelCase		)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase		)    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       Union[str, Any]		):
       _A					   =   list(self.get_tokenizer().get_vocab().keys()		)
       self.assertEqual(vocab_keys[0]    ,							'<pad>'		)
       self.assertEqual(vocab_keys[1]    ,							'<unk>'		)
       self.assertEqual(vocab_keys[-1]    ,							'[PAD]'		)
       self.assertEqual(len(_UpperCAmelCase		)    ,							30_001		)

 def 			lowerCAmelCase_							( self       :       Optional[Any]		):
       self.assertEqual(self.get_tokenizer().vocab_size    ,							30_000		)

 def 			lowerCAmelCase_							( self       :       str		):
       # fmt: off
       _A					   =   ' \tHeLLo!how  \n Are yoU?  '
       _A					   =   ['▁hello', '!', 'how', '▁are', '▁you', '?']
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'		)
 def 			lowerCAmelCase_							( self       :       Optional[int]		):
       pass

 @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'		)
 def 			lowerCAmelCase_							( self       :       Dict		):
       pass

 def 			lowerCAmelCase_							( self       :       str		):
       # fmt: off
       _A					   =   'I was born in 92000, and this is falsé.'
       _A					   =   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       int		):
       # fmt: off
       _A					   =   'I was born in 92000, and this is falsé.'
       _A					   =   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       Union[str, Any]		):
       # fmt: off
       _A					   =   'I was born in 92000, and this is falsé.'
       _A					   =   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       List[str]		):
       # fmt: off
       _A					   =   'I was born in 92000, and this is falsé.'
       _A					   =   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       Optional[int]		):
       # fmt: off
       _A					   =   ' \tHeLLo!how  \n Are yoU?  '
       _A					   =   ['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
       # fmt: on

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							do_lower_case=_UpperCAmelCase    ,							split_by_punct=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)

       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       Dict		):
       _A					   =   self.get_tokenizer()
       _A					   =   self.get_rust_tokenizer()

       _A					   =   'I was born in 92000, and this is falsé.'

       _A					   =   tokenizer.convert_ids_to_tokens(tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       _A					   =   rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   self.get_rust_tokenizer()
       _A					   =   tokenizer.encode(_UpperCAmelCase		)
       _A					   =   rust_tokenizer.encode(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       List[Any]		):
       _A					   =   'This is a test'
       _A					   =   [13, 1, 4_398, 25, 21, 1_289]
       _A					   =   ['▁', 'T', 'his', '▁is', '▁a', '▁test']
       _A					   =   ['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

       _A					   =   DebertaVaTokenizer(_UpperCAmelCase    ,							keep_accents=_UpperCAmelCase		)
       _A					   =   DebertaVaTokenizerFast(_UpperCAmelCase    ,							keep_accents=_UpperCAmelCase		)

       _A					   =   tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   tokenizer.tokenize(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   rust_tokenizer.tokenize(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       # fmt: off
       _A					   =   'I was born in 92000, and this is falsé.'
       _A					   =   [13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
       _A					   =   ['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
       _A					   =   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
       # fmt: on

       _A					   =   tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   tokenizer.tokenize(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   tokenizer.convert_ids_to_tokens(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

       _A					   =   rust_tokenizer.encode(_UpperCAmelCase    ,							add_special_tokens=_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   rust_tokenizer.tokenize(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)
       _A					   =   rust_tokenizer.convert_ids_to_tokens(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase    ,							_UpperCAmelCase		)

 def 			lowerCAmelCase_							( self       :       int		):
       _A					   =   DebertaVaTokenizer(_UpperCAmelCase		)

       _A					   =   tokenizer.encode('sequence builders'		)
       _A					   =   tokenizer.encode('multi-sequence build'		)

       _A					   =   tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase		)
       _A					   =   tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase    ,							_UpperCAmelCase		)

       self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]    ,							_UpperCAmelCase		)
       self.assertEqual(
           [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]    ,							_UpperCAmelCase    ,							)



 @slow
 def 			lowerCAmelCase_							( self       :       Tuple		):
       # fmt: off
       _A					   =   {'input_ids': [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_UpperCAmelCase    ,							model_name='microsoft/deberta-v2-xlarge'    ,							revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'    ,							)








import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


_UpperCAmelCase				:       Optional[Any]	   =				logging.getLogger(__name__)

@dataclass(frozen=__UpperCamelCase			)
class    lowerCAmelCase							:
 UpperCAmelCase__													=		42
 UpperCAmelCase__													=		42
 UpperCAmelCase__													=		None
 UpperCAmelCase__													=		None
 UpperCAmelCase__													=		None


@dataclass(frozen=__UpperCamelCase			)
class    lowerCAmelCase							:
 UpperCAmelCase__													=		42
 UpperCAmelCase__													=		None
 UpperCAmelCase__													=		None
 UpperCAmelCase__													=		None
 UpperCAmelCase__													=		None


if is_torch_available():
    import torch
    from torch.utils.data import Dataset

    class    lowerCAmelCase							(					__UpperCamelCase			):
     UpperCAmelCase__													=		42
     def __init__(      self  :      int			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      PreTrainedTokenizer			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[int] = None			,     UpperCAmelCase  :      List[str]=False			,     UpperCAmelCase  :      bool = False			,     )			->	List[str]:
         lowerCamelCase__    :      int					 =		hans_processors[task]()

         lowerCamelCase__    :      Optional[Any]					 =		os.path.join(
             UpperCAmelCase			,     'cached_{}_{}_{}_{}'.format(
                 'dev' if evaluate else 'train'			,     tokenizer.__class__.__name__			,     str(UpperCAmelCase	)			,     UpperCAmelCase			,     )			,     )
         lowerCamelCase__    :      int					 =		processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             lowerCamelCase__ ,							lowerCamelCase__    :      Union[str, Any]					 =		label_list[2], label_list[1]
         lowerCamelCase__    :      List[str]					 =		label_list

         # Make sure only the first process in distributed training processes the dataset,
         # and the others will use the cache.
         lowerCamelCase__    :      str					 =		cached_features_file + '.lock'
         with FileLock(UpperCAmelCase	):
             if os.path.exists(UpperCAmelCase	) and not overwrite_cache:
                 logger.info(F"""Loading features from cached file {cached_features_file}"""	)
                 lowerCamelCase__    :      int					 =		torch.load(UpperCAmelCase	)
             else:
                 logger.info(F"""Creating features from dataset file at {data_dir}"""	)

                 lowerCamelCase__    :      str					 =		(
                     processor.get_dev_examples(UpperCAmelCase	) if evaluate else processor.get_train_examples(UpperCAmelCase	)
                 )

                 logger.info('Training examples: %s'			,     len(UpperCAmelCase	)	)
                 lowerCamelCase__    :      Dict					 =		hans_convert_examples_to_features(UpperCAmelCase			,     UpperCAmelCase			,     UpperCAmelCase			,     UpperCAmelCase	)
                 logger.info('Saving features into cached file %s'			,     UpperCAmelCase	)
                 torch.save(self.features			,     UpperCAmelCase	)
     def __len__(      self  :      Optional[int]	)			->	Optional[Any]:
         return len(self.features	)
     def __getitem__(      self  :      Tuple			,     UpperCAmelCase  :      Dict	)			->	InputFeatures:
         return self.features[i]







     def      A_	(      self  :      int	)			->	int:
         return self.label_list


if is_tf_available():
    import tensorflow as tf

    class    lowerCAmelCase							:
     UpperCAmelCase__													=		42
     def __init__(      self  :      Optional[Any]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      PreTrainedTokenizer			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      Optional[int] = 128			,     UpperCAmelCase  :      Any=False			,     UpperCAmelCase  :      bool = False			,     )			->	Union[str, Any]:
         lowerCamelCase__    :      Any					 =		hans_processors[task]()
         lowerCamelCase__    :      Optional[Any]					 =		processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		label_list[2], label_list[1]
         lowerCamelCase__    :      Optional[int]					 =		label_list

         lowerCamelCase__    :      int					 =		processor.get_dev_examples(UpperCAmelCase	) if evaluate else processor.get_train_examples(UpperCAmelCase	)
         lowerCamelCase__    :      Union[str, Any]					 =		hans_convert_examples_to_features(UpperCAmelCase			,     UpperCAmelCase			,     UpperCAmelCase			,     UpperCAmelCase	)

         def gen():
             for ex_index, ex in tqdm.tqdm(enumerate(self.features	)			,     desc='convert examples to features'	):
                 if ex_index % 10000 == 0:
                     logger.info('Writing example %d of %d' % (ex_index, len(UpperCAmelCase	))	)

                 yield (
                     {
                         "example_id": 0,
                         "input_ids": ex.input_ids,
                         "attention_mask": ex.attention_mask,
                         "token_type_ids": ex.token_type_ids,
                     },
                     ex.label,
                 )

         lowerCamelCase__    :      Optional[int]					 =		tf.data.Dataset.from_generator(
             UpperCAmelCase			,     (
                 {
                     'example_id': tf.intaa,
                     'input_ids': tf.intaa,
                     'attention_mask': tf.intaa,
                     'token_type_ids': tf.intaa,
                 },
                 tf.intaa,
             )			,     (
                 {
                     'example_id': tf.TensorShape([]	),
                     'input_ids': tf.TensorShape([None, None]	),
                     'attention_mask': tf.TensorShape([None, None]	),
                     'token_type_ids': tf.TensorShape([None, None]	),
                 },
                 tf.TensorShape([]	),
             )			,     )
     def      A_	(      self  :      Any	)			->	Any:
         return self.dataset
     def __len__(      self  :      Tuple	)			->	int:
         return len(self.features	)
     def __getitem__(      self  :      List[str]			,     UpperCAmelCase  :      Any	)			->	InputFeatures:
         return self.features[i]







     def      A_	(      self  :      Dict	)			->	str:
         return self.label_list

class    lowerCAmelCase							(					__UpperCamelCase			):
 def      A_	(      self  :      int			,     UpperCAmelCase  :      List[Any]	)			->	int:
     return self._create_examples(self._read_tsv(os.path.join(UpperCAmelCase			,     'heuristics_train_set.txt'	)	)			,     'train'	)
 def      A_	(      self  :      Any			,     UpperCAmelCase  :      int	)			->	List[Any]:
     return self._create_examples(self._read_tsv(os.path.join(UpperCAmelCase			,     'heuristics_evaluation_set.txt'	)	)			,     'dev'	)
 def      A_	(      self  :      Any	)			->	List[Any]:
     return ["contradiction", "entailment", "neutral"]







 def      A_	(      self  :      Optional[int]			,     UpperCAmelCase  :      str			,     UpperCAmelCase  :      List[str]	)			->	List[str]:
     lowerCamelCase__    :      List[str]					 =		[]
     for i, line in enumerate(UpperCAmelCase	):
         if i == 0:
             continue
         lowerCamelCase__    :      Tuple					 =		'%s-%s' % (set_type, line[0])
         lowerCamelCase__    :      str					 =		line[5]
         lowerCamelCase__    :      Dict					 =		line[6]
         lowerCamelCase__    :      int					 =		line[7][2:] if line[7].startswith('ex'	) else line[7]
         lowerCamelCase__    :      Dict					 =		line[0]
         examples.append(InputExample(guid=UpperCAmelCase			,     text_a=UpperCAmelCase			,     text_b=UpperCAmelCase			,     label=UpperCAmelCase			,     pairID=UpperCAmelCase	)	)
     return examples


def        SCREAMING_SNAKE_CASE      (       _UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	_UpperCAmelCase						,	)			->     Optional[int]:
    lowerCamelCase__    :      int					 =		{label: i for i, label in enumerate(_UpperCAmelCase )}

    lowerCamelCase__    :      List[Any]					 =		[]
    for ex_index, example in tqdm.tqdm(enumerate(_UpperCAmelCase )						,	desc='convert examples to features' ):
        if ex_index % 1_0000 == 0:
            logger.info('Writing example %d' % (ex_index) )

        lowerCamelCase__    :      List[Any]					 =		tokenizer(
            example.text_a						,	example.text_b						,	add_special_tokens=_UpperCAmelCase						,	max_length=_UpperCAmelCase						,	padding='max_length'						,	truncation=_UpperCAmelCase						,	return_overflowing_tokens=_UpperCAmelCase						,	)

        lowerCamelCase__    :      List[str]					 =		label_map[example.label] if example.label in label_map else 0

        lowerCamelCase__    :      Optional[int]					 =		int(example.pairID )

        features.append(InputFeatures(**_UpperCAmelCase						,	label=_UpperCAmelCase						,	pairID=_UpperCAmelCase ) )

    for i, example in enumerate(examples[:5] ):
        logger.info('*** Example ***' )
        logger.info(F"""guid: {example}""" )
        logger.info(F"""features: {features[i]}""" )

    return features


_UpperCAmelCase				:       str	   =				{
    """hans""": 3,
}


_UpperCAmelCase				:       List[Any]	   =				{
    """hans""": HansProcessor,
}





