




import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowerCamelCase				:		Any     =   logging.getLogger(__name__)







@dataclass(frozen=_a   )
class   A:



 '''simple docstring'''


 UpperCamelCase						   =						42
 UpperCamelCase						   =						42
 UpperCamelCase						   =						None
 UpperCamelCase						   =						None
 UpperCamelCase						   =						None







@dataclass(frozen=_a   )
class   A:



 '''simple docstring'''


 UpperCamelCase						   =						42
 UpperCamelCase						   =						None
 UpperCamelCase						   =						None
 UpperCamelCase						   =						None
 UpperCamelCase						   =						None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset







 class   A(						_a   ):



  '''simple docstring'''


  UpperCamelCase						   =						42

  def __init__(				self					:  Optional[int]						,							A_					:  str						,							A_					:  PreTrainedTokenizer						,							A_					:  str						,							A_					:  Optional[int] = None						,							A_					:  Dict=False						,							A_					:  bool = False						,							)   ->	Dict:

        """simple docstring"""






        lowerCamelCase_		     =    hans_processors[task]()

        lowerCamelCase_		     =    os.path.join(
            snake_case_						,							'cached_{}_{}_{}_{}'.format(
                'dev' if evaluate else 'train'						,							tokenizer.__class__.__name__						,							str(snake_case_  )						,							snake_case_						,							)						,							)
        lowerCamelCase_		     =    processor.get_labels()
        if tokenizer.__class__ in (
            RobertaTokenizer,
            RobertaTokenizerFast,
            XLMRobertaTokenizer,
            BartTokenizer,
            BartTokenizerFast,
        ):
              # HACK(label indices are swapped in RoBERTa pretrained model)
              lowerCamelCase_		     =    label_list[2], label_list[1]
        lowerCamelCase_		     =    label_list

        # Make sure only the first process in distributed training processes the dataset,
        # and the others will use the cache.
        lowerCamelCase_		     =    cached_features_file + """.lock"""
        with FileLock(snake_case_  ):
              if os.path.exists(snake_case_  ) and not overwrite_cache:
                    logger.info(f"""Loading features from cached file {cached_features_file}"""  )
                    lowerCamelCase_		     =    torch.load(snake_case_  )
              else:
                    logger.info(f"""Creating features from dataset file at {data_dir}"""  )

                    lowerCamelCase_		     =    (
                        processor.get_dev_examples(snake_case_  ) if evaluate else processor.get_train_examples(snake_case_  )
                    )

                    logger.info('Training examples: %s'						,							len(snake_case_  )  )
                    lowerCamelCase_		     =    hans_convert_examples_to_features(snake_case_						,							snake_case_						,							snake_case_						,							snake_case_  )
                    logger.info('Saving features into cached file %s'						,							snake_case_  )
                    torch.save(self.features						,							snake_case_  )

  def __len__(				self					:  Any  )   ->	Any:

        """simple docstring"""






        return len(self.features  )

  def __getitem__(				self					:  int						,							A_					:  Optional[int]  )   ->	List[Any]:

        """simple docstring"""






        return self.features[i]





  def 				a__ (				self					:  Optional[int]  )   ->	List[Any]:

        """simple docstring"""






        return self.label_list


if is_tf_available():
 import tensorflow as tf







 class   A:



  '''simple docstring'''


  UpperCamelCase						   =						42

  def __init__(				self					:  Optional[int]						,							A_					:  str						,							A_					:  PreTrainedTokenizer						,							A_					:  str						,							A_					:  Optional[int] = 128						,							A_					:  Union[str, Any]=False						,							A_					:  bool = False						,							)   ->	Optional[Any]:

        """simple docstring"""






        lowerCamelCase_		     =    hans_processors[task]()
        lowerCamelCase_		     =    processor.get_labels()
        if tokenizer.__class__ in (
            RobertaTokenizer,
            RobertaTokenizerFast,
            XLMRobertaTokenizer,
            BartTokenizer,
            BartTokenizerFast,
        ):
              # HACK(label indices are swapped in RoBERTa pretrained model)
              lowerCamelCase_		     =    label_list[2], label_list[1]
        lowerCamelCase_		     =    label_list

        lowerCamelCase_		     =    processor.get_dev_examples(snake_case_  ) if evaluate else processor.get_train_examples(snake_case_  )
        lowerCamelCase_		     =    hans_convert_examples_to_features(snake_case_						,							snake_case_						,							snake_case_						,							snake_case_  )

        def gen():
              for ex_index, ex in tqdm.tqdm(enumerate(self.features  )						,							desc='convert examples to features'  ):
                    if ex_index % 10000 == 0:
                          logger.info('Writing example %d of %d' % (ex_index, len(snake_case_  ))  )

                    yield (
                        {
                            "example_id": 0,
                            "input_ids": ex.input_ids,
                            "attention_mask": ex.attention_mask,
                            "token_type_ids": ex.token_type_ids,
                        },
                        ex.label,
                    )

        lowerCamelCase_		     =    tf.data.Dataset.from_generator(
            snake_case_						,							(
                {
                    'example_id': tf.intaa,
                    'input_ids': tf.intaa,
                    'attention_mask': tf.intaa,
                    'token_type_ids': tf.intaa,
                },
                tf.intaa,
            )						,							(
                {
                    'example_id': tf.TensorShape([]  ),
                    'input_ids': tf.TensorShape([None, None]  ),
                    'attention_mask': tf.TensorShape([None, None]  ),
                    'token_type_ids': tf.TensorShape([None, None]  ),
                },
                tf.TensorShape([]  ),
            )						,							)

  def 				a__ (				self					:  Optional[Any]  )   ->	str:

        """simple docstring"""






        return self.dataset

  def __len__(				self					:  List[str]  )   ->	str:

        """simple docstring"""






        return len(self.features  )

  def __getitem__(				self					:  List[Any]						,							A_					:  Dict  )   ->	Tuple:

        """simple docstring"""






        return self.features[i]





  def 				a__ (				self					:  Dict  )   ->	Any:

        """simple docstring"""






        return self.label_list







class   A(						_a   ):



 '''simple docstring'''


 def 				a__ (				self					:  Dict						,							A_					:  Tuple  )   ->	Any:

       """simple docstring"""






       return self._create_examples(self._read_tsv(os.path.join(snake_case_						,							'heuristics_train_set.txt'  )  )						,							'train'  )

 def 				a__ (				self					:  Optional[int]						,							A_					:  Union[str, Any]  )   ->	str:

       """simple docstring"""






       return self._create_examples(self._read_tsv(os.path.join(snake_case_						,							'heuristics_evaluation_set.txt'  )  )						,							'dev'  )

 def 				a__ (				self					:  List[Any]  )   ->	str:

       """simple docstring"""






       return ["contradiction", "entailment", "neutral"]





 def 				a__ (				self					:  List[Any]						,							A_					:  int						,							A_					:  Optional[Any]  )   ->	Dict:

       """simple docstring"""






       lowerCamelCase_		     =    []
       for i, line in enumerate(snake_case_  ):
             if i == 0:
                   continue
             lowerCamelCase_		     =    """%s-%s""" % (set_type, line[0])
             lowerCamelCase_		     =    line[5]
             lowerCamelCase_		     =    line[6]
             lowerCamelCase_		     =    line[7][2:] if line[7].startswith('ex'  ) else line[7]
             lowerCamelCase_		     =    line[0]
             examples.append(InputExample(guid=snake_case_						,							text_a=snake_case_						,							text_b=snake_case_						,							label=snake_case_						,							pairID=snake_case_  )  )
       return examples

def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       Any	,							lowercase	:       List[Any]	,							lowercase	:       Tuple	,							lowercase	:       Tuple	,							):


      '''simple docstring'''



      lowerCamelCase_		     =    {label: i for i, label in enumerate(_lowerCAmelCase     )}

      lowerCamelCase_		     =    []
      for ex_index, example in tqdm.tqdm(enumerate(_lowerCAmelCase     )	,							desc='convert examples to features'     ):
            if ex_index % 1_00_00 == 0:
                  logger.info('Writing example %d' % (ex_index)     )

            lowerCamelCase_		     =    tokenizer(
                example.text_a	,							example.text_b	,							add_special_tokens=_lowerCAmelCase	,							max_length=_lowerCAmelCase	,							padding='max_length'	,							truncation=_lowerCAmelCase	,							return_overflowing_tokens=_lowerCAmelCase	,							)

            lowerCamelCase_		     =    label_map[example.label] if example.label in label_map else 0

            lowerCamelCase_		     =    int(example.pairID     )

            features.append(InputFeatures(**_lowerCAmelCase	,							label=_lowerCAmelCase	,							pairID=_lowerCAmelCase     )     )

      for i, example in enumerate(examples[:5]     ):
            logger.info('*** Example ***'     )
            logger.info(f"""guid: {example}"""     )
            logger.info(f"""features: {features[i]}"""     )

      return features


lowerCamelCase				:		Optional[Any]     =   {
    "hans": 3,
}






lowerCamelCase				:		Optional[int]     =   {
    "hans": HansProcessor,
}








'''simple docstring'''





import string
from math import logaa





def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase		)		-> int:
      snake_case__  :    List[str]    						=      document.translate(
          str.maketrans(""""""     ,   """"""     ,   string.punctuation		)		).replace("""\n"""     ,   """"""		)
      snake_case__  :    List[str]    						=      document_without_punctuation.split(""" """		)  # word tokenization
      return len([word for word in tokenize_document if word.lower() == term.lower()]		)





def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase		)		-> tuple[int, int]:
      snake_case__  :    Dict    						=      corpus.lower().translate(
          str.maketrans(""""""     ,   """"""     ,   string.punctuation		)		)  # strip all punctuation and replace it with ''
      snake_case__  :    Any    						=      corpus_without_punctuation.split("""\n"""		)
      snake_case__  :    int    						=      term.lower()
      return (len([doc for doc in docs if term in doc]		), len(_lowerCAmelCase		))





def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase     ,   _lowerCAmelCase=False		)		-> float:
      if smoothing:
            if n == 0:
                  raise ValueError("""log10(0) is undefined."""		)
            return round(1 + logaa(n / (1 + df)		)     ,   3		)

      if df == 0:
            raise ZeroDivisionError("""df must be > 0"""		)
      elif n == 0:
            raise ValueError("""log10(0) is undefined."""		)
      return round(logaa(n / df		)     ,   3		)





def 						__snake_case(				_lowerCAmelCase     ,   _lowerCAmelCase		)		-> float:
      return round(tf * idf     ,   3		)
