


'''simple docstring'''

import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowerCamelCase							:   Optional[Any]  						=	logging.getLogger(__name__)
@dataclass(frozen=lowercase_     )
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     str
      lowerCAmelCase__					:     str
      lowerCAmelCase__					:     Optional[str]  					=							None
      lowerCAmelCase__					:     Optional[str]  					=							None
      lowerCAmelCase__					:     Optional[str]  					=							None






@dataclass(frozen=lowercase_     )
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     List[int]
      lowerCAmelCase__					:     Optional[List[int]]  					=							None
      lowerCAmelCase__					:     Optional[List[int]]  					=							None
      lowerCAmelCase__					:     Optional[Union[int, float]]  					=							None
      lowerCAmelCase__					:     Optional[int]  					=							None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset
 class   __lowerCAmelCase						(lowercase_     ):





       '''simple docstring'''






       lowerCAmelCase__					:     List[InputFeatures]



       def __init__(self    :    Optional[int]   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    PreTrainedTokenizer   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[int] = None   ,						UpperCamelCase    :    List[Any]=False   ,						UpperCamelCase    :    bool = False   ,						):

              '''simple docstring'''







              lowercase__      			= hans_processors[task]()

              lowercase__      			= os.path.join(
                  UpperCamelCase   ,						'''cached_{}_{}_{}_{}'''.format(
                      '''dev''' if evaluate else '''train'''   ,						tokenizer.__class__.__name__   ,						str(UpperCamelCase )   ,						UpperCamelCase   ,						)   ,						)
              lowercase__      			= processor.get_labels()
              if tokenizer.__class__ in (
                  RobertaTokenizer,
                  RobertaTokenizerFast,
                  XLMRobertaTokenizer,
                  BartTokenizer,
                  BartTokenizerFast,
              ):
                     # HACK(label indices are swapped in RoBERTa pretrained model)
                     lowercase__			,lowercase__      			= label_list[2], label_list[1]
              lowercase__      			= label_list

              # Make sure only the first process in distributed training processes the dataset,
              # and the others will use the cache.
              lowercase__      			= cached_features_file + '''.lock'''
              with FileLock(UpperCamelCase ):
                     if os.path.exists(UpperCamelCase ) and not overwrite_cache:
                            logger.info(f"Loading features from cached file {cached_features_file}" )
                            lowercase__      			= torch.load(UpperCamelCase )
                     else:
                            logger.info(f"Creating features from dataset file at {data_dir}" )

                            lowercase__      			= (
                                processor.get_dev_examples(UpperCamelCase ) if evaluate else processor.get_train_examples(UpperCamelCase )
                            )

                            logger.info('''Training examples: %s'''   ,						len(UpperCamelCase ) )
                            lowercase__      			= hans_convert_examples_to_features(UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase )
                            logger.info('''Saving features into cached file %s'''   ,						UpperCamelCase )
                            torch.save(self.features   ,						UpperCamelCase )



       def __len__(self    :    Optional[int] ):

              '''simple docstring'''







              return len(self.features )



       def __getitem__(self    :    Tuple   ,						UpperCamelCase    :    str ):

              '''simple docstring'''







              return self.features[i]






       def       UpperCamelCase__		(self    :    str ):

              '''simple docstring'''







              return self.label_list


if is_tf_available():
 import tensorflow as tf
 class   __lowerCAmelCase						:





       '''simple docstring'''






       lowerCAmelCase__					:     List[InputFeatures]



       def __init__(self    :    List[Any]   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    PreTrainedTokenizer   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[int] = 128   ,						UpperCamelCase    :    Union[str, Any]=False   ,						UpperCamelCase    :    bool = False   ,						):

              '''simple docstring'''







              lowercase__      			= hans_processors[task]()
              lowercase__      			= processor.get_labels()
              if tokenizer.__class__ in (
                  RobertaTokenizer,
                  RobertaTokenizerFast,
                  XLMRobertaTokenizer,
                  BartTokenizer,
                  BartTokenizerFast,
              ):
                     # HACK(label indices are swapped in RoBERTa pretrained model)
                     lowercase__			,lowercase__      			= label_list[2], label_list[1]
              lowercase__      			= label_list

              lowercase__      			= processor.get_dev_examples(UpperCamelCase ) if evaluate else processor.get_train_examples(UpperCamelCase )
              lowercase__      			= hans_convert_examples_to_features(UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase )

              def gen():
                     for ex_index, ex in tqdm.tqdm(enumerate(self.features )   ,						desc='''convert examples to features''' ):
                            if ex_index % 10000 == 0:
                                   logger.info('''Writing example %d of %d''' % (ex_index, len(UpperCamelCase )) )

                            yield (
                                {
                                    "example_id": 0,
                                    "input_ids": ex.input_ids,
                                    "attention_mask": ex.attention_mask,
                                    "token_type_ids": ex.token_type_ids,
                                },
                                ex.label,
                            )

              lowercase__      			= tf.data.Dataset.from_generator(
                  UpperCamelCase   ,						(
                      {
                          '''example_id''': tf.intaa,
                          '''input_ids''': tf.intaa,
                          '''attention_mask''': tf.intaa,
                          '''token_type_ids''': tf.intaa,
                      },
                      tf.intaa,
                  )   ,						(
                      {
                          '''example_id''': tf.TensorShape([] ),
                          '''input_ids''': tf.TensorShape([None, None] ),
                          '''attention_mask''': tf.TensorShape([None, None] ),
                          '''token_type_ids''': tf.TensorShape([None, None] ),
                      },
                      tf.TensorShape([] ),
                  )   ,						)



       def       UpperCamelCase__		(self    :    Optional[int] ):

              '''simple docstring'''







              return self.dataset



       def __len__(self    :    List[Any] ):

              '''simple docstring'''







              return len(self.features )



       def __getitem__(self    :    Union[str, Any]   ,						UpperCamelCase    :    Tuple ):

              '''simple docstring'''







              return self.features[i]






       def       UpperCamelCase__		(self    :    Union[str, Any] ):

              '''simple docstring'''







              return self.label_list
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    Any ):

             '''simple docstring'''







             return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase   ,						'''heuristics_train_set.txt''' ) )   ,						'''train''' )



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    Optional[Any] ):

             '''simple docstring'''







             return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase   ,						'''heuristics_evaluation_set.txt''' ) )   ,						'''dev''' )



      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             return ["contradiction", "entailment", "neutral"]






      def       UpperCamelCase__		(self    :    Optional[Any]   ,						UpperCamelCase    :    int   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= []
             for i, line in enumerate(UpperCamelCase ):
                    if i == 0:
                           continue
                    lowercase__      			= '''%s-%s''' % (set_type, line[0])
                    lowercase__      			= line[5]
                    lowercase__      			= line[6]
                    lowercase__      			= line[7][2:] if line[7].startswith('''ex''' ) else line[7]
                    lowercase__      			= line[0]
                    examples.append(InputExample(guid=UpperCamelCase   ,						text_a=UpperCamelCase   ,						text_b=UpperCamelCase   ,						label=UpperCamelCase   ,						pairID=UpperCamelCase ) )
             return examples






def _SCREAMING_SNAKE_CASE      (A  , A  , A  , A  , )       ->       int:






       """simple docstring"""







       lowercase__      			= {label: i for i, label in enumerate(A       )}

       lowercase__      			= []
       for ex_index, example in tqdm.tqdm(enumerate(A       )  , desc='''convert examples to features'''       ):
              if ex_index % 10_000 == 0:
                     logger.info('''Writing example %d''' % (ex_index)       )

              lowercase__      			= tokenizer(
                  example.text_a  , example.text_b  , add_special_tokens=A  , max_length=A  , padding='''max_length'''  , truncation=A  , return_overflowing_tokens=A  , )

              lowercase__      			= label_map[example.label] if example.label in label_map else 0

              lowercase__      			= int(example.pairID       )

              features.append(InputFeatures(**A  , label=A  , pairID=A       )       )

       for i, example in enumerate(examples[:5]       ):
              logger.info('''*** Example ***'''       )
              logger.info(f"guid: {example}"       )
              logger.info(f"features: {features[i]}"       )

       return features


lowerCamelCase							:   Tuple  						=	{
    'hans': 3,
}

lowerCamelCase							:   str  						=	{
    'hans': HansProcessor,
}








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase							:   str  						=	{
    'configuration_rag': ['RagConfig'],
    'retrieval_rag': ['RagRetriever'],
    'tokenization_rag': ['RagTokenizer'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Dict  						=	[
     'RagModel',
     'RagPreTrainedModel',
     'RagSequenceForGeneration',
     'RagTokenForGeneration',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Optional[int]  						=	[
     'TFRagModel',
     'TFRagPreTrainedModel',
     'TFRagSequenceForGeneration',
     'TFRagTokenForGeneration',
 ]


if TYPE_CHECKING:
 from .configuration_rag import RagConfig
 from .retrieval_rag import RagRetriever
 from .tokenization_rag import RagTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_rag import (
      TFRagModel,
      TFRagPreTrainedModel,
      TFRagSequenceForGeneration,
      TFRagTokenForGeneration,
  )

else:
 import sys

 lowerCamelCase							:   Tuple  						=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




