


'''simple docstring'''

import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


lowerCamelCase							:   str  						=	logging.getLogger(__name__)
@dataclass
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     str
      lowerCAmelCase__					:     List[str]
      lowerCAmelCase__					:     Optional[List[str]]






@dataclass
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     List[int]
      lowerCAmelCase__					:     List[int]
      lowerCAmelCase__					:     Optional[List[int]]  					=							None
      lowerCAmelCase__					:     Optional[List[int]]  					=							None






class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Dict  					=							"""train"""
      lowerCAmelCase__					:     Any  					=							"""dev"""
      lowerCAmelCase__					:     List[str]  					=							"""test"""






class   __lowerCAmelCase						:





      '''simple docstring'''






      @staticmethod
      def       UpperCamelCase__		(UpperCamelCase    :    Dict   ,						UpperCamelCase    :    Union[Split, str] ):

             '''simple docstring'''







             raise NotImplementedError



      @staticmethod
      def       UpperCamelCase__		(UpperCamelCase    :    str ):

             '''simple docstring'''







             raise NotImplementedError






      @staticmethod
      def       UpperCamelCase__		(UpperCamelCase    :    List[InputExample]   ,						UpperCamelCase    :    List[str]   ,						UpperCamelCase    :    int   ,						UpperCamelCase    :    PreTrainedTokenizer   ,						UpperCamelCase    :    str=False   ,						UpperCamelCase    :    Dict="[CLS]"   ,						UpperCamelCase    :    Optional[int]=1   ,						UpperCamelCase    :    int="[SEP]"   ,						UpperCamelCase    :    Optional[Any]=False   ,						UpperCamelCase    :    Union[str, Any]=False   ,						UpperCamelCase    :    Tuple=0   ,						UpperCamelCase    :    Dict=0   ,						UpperCamelCase    :    List[Any]=-100   ,						UpperCamelCase    :    int=0   ,						UpperCamelCase    :    str=True   ,						):

             '''simple docstring'''







             lowercase__      			= {label: i for i, label in enumerate(UpperCamelCase )}

             lowercase__      			= []
             for ex_index, example in enumerate(UpperCamelCase ):
                    if ex_index % 10000 == 0:
                           logger.info('''Writing example %d of %d'''   ,						UpperCamelCase   ,						len(UpperCamelCase ) )

                    lowercase__      			= []
                    lowercase__      			= []
                    for word, label in zip(example.words   ,						example.labels ):
                           lowercase__      			= tokenizer.tokenize(UpperCamelCase )

                           # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
                           if len(UpperCamelCase ) > 0:
                                  tokens.extend(UpperCamelCase )
                                  # Use the real label id for the first token of the word, and padding ids for the remaining tokens
                                  label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(UpperCamelCase ) - 1) )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
                    lowercase__      			= tokenizer.num_special_tokens_to_add()
                    if len(UpperCamelCase ) > max_seq_length - special_tokens_count:
                           lowercase__      			= tokens[: (max_seq_length - special_tokens_count)]
                           lowercase__      			= label_ids[: (max_seq_length - special_tokens_count)]

                    # The convention in BERT is:
                    # (a) For sequence pairs:
                    #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
                    #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
                    # (b) For single sequences:
                    #  tokens:   [CLS] the dog is hairy . [SEP]
                    #  type_ids:   0   0   0   0  0     0   0
                    #
                    # Where "type_ids" are used to indicate whether this is the first
                    # sequence or the second sequence. The embedding vectors for `type=0` and
                    # `type=1` were learned during pre-training and are added to the wordpiece
                    # embedding vector (and position vector). This is not *strictly* necessary
                    # since the [SEP] token unambiguously separates the sequences, but it makes
                    # it easier for the model to learn the concept of sequences.
                    #
                    # For classification tasks, the first vector (corresponding to [CLS]) is
                    # used as the "sentence vector". Note that this only makes sense because
                    # the entire model is fine-tuned.
                    tokens += [sep_token]
                    label_ids += [pad_token_label_id]
                    if sep_token_extra:
                           # roberta uses an extra separator b/w pairs of sentences
                           tokens += [sep_token]
                           label_ids += [pad_token_label_id]
                    lowercase__      			= [sequence_a_segment_id] * len(UpperCamelCase )

                    if cls_token_at_end:
                           tokens += [cls_token]
                           label_ids += [pad_token_label_id]
                           segment_ids += [cls_token_segment_id]
                    else:
                           lowercase__      			= [cls_token] + tokens
                           lowercase__      			= [pad_token_label_id] + label_ids
                           lowercase__      			= [cls_token_segment_id] + segment_ids

                    lowercase__      			= tokenizer.convert_tokens_to_ids(UpperCamelCase )

                    # The mask has 1 for real tokens and 0 for padding tokens. Only real
                    # tokens are attended to.
                    lowercase__      			= [1 if mask_padding_with_zero else 0] * len(UpperCamelCase )

                    # Zero-pad up to the sequence length.
                    lowercase__      			= max_seq_length - len(UpperCamelCase )
                    if pad_on_left:
                           lowercase__      			= ([pad_token] * padding_length) + input_ids
                           lowercase__      			= ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
                           lowercase__      			= ([pad_token_segment_id] * padding_length) + segment_ids
                           lowercase__      			= ([pad_token_label_id] * padding_length) + label_ids
                    else:
                           input_ids += [pad_token] * padding_length
                           input_mask += [0 if mask_padding_with_zero else 1] * padding_length
                           segment_ids += [pad_token_segment_id] * padding_length
                           label_ids += [pad_token_label_id] * padding_length

                    assert len(UpperCamelCase ) == max_seq_length
                    assert len(UpperCamelCase ) == max_seq_length
                    assert len(UpperCamelCase ) == max_seq_length
                    assert len(UpperCamelCase ) == max_seq_length

                    if ex_index < 5:
                           logger.info('''*** Example ***''' )
                           logger.info('''guid: %s'''   ,						example.guid )
                           logger.info('''tokens: %s'''   ,						''' '''.join([str(UpperCamelCase ) for x in tokens] ) )
                           logger.info('''input_ids: %s'''   ,						''' '''.join([str(UpperCamelCase ) for x in input_ids] ) )
                           logger.info('''input_mask: %s'''   ,						''' '''.join([str(UpperCamelCase ) for x in input_mask] ) )
                           logger.info('''segment_ids: %s'''   ,						''' '''.join([str(UpperCamelCase ) for x in segment_ids] ) )
                           logger.info('''label_ids: %s'''   ,						''' '''.join([str(UpperCamelCase ) for x in label_ids] ) )

                    if "token_type_ids" not in tokenizer.model_input_names:
                           lowercase__      			= None

                    features.append(
                        InputFeatures(
                            input_ids=UpperCamelCase   ,						attention_mask=UpperCamelCase   ,						token_type_ids=UpperCamelCase   ,						label_ids=UpperCamelCase ) )
             return features


if is_torch_available():
 import torch
 from torch import nn
 from torch.utils.data import Dataset
 class   __lowerCAmelCase						(lowercase_     ):





       '''simple docstring'''






       lowerCAmelCase__					:     List[InputFeatures]
       lowerCAmelCase__					:     int  					=							nn.CrossEntropyLoss().ignore_index



       def __init__(self    :    str   ,						UpperCamelCase    :    TokenClassificationTask   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    PreTrainedTokenizer   ,						UpperCamelCase    :    List[str]   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[int] = None   ,						UpperCamelCase    :    int=False   ,						UpperCamelCase    :    Split = Split.train   ,						):

              '''simple docstring'''







              lowercase__      			= os.path.join(
                  UpperCamelCase   ,						'''cached_{}_{}_{}'''.format(mode.value   ,						tokenizer.__class__.__name__   ,						str(UpperCamelCase ) )   ,						)

              # Make sure only the first process in distributed training processes the dataset,
              # and the others will use the cache.
              lowercase__      			= cached_features_file + '''.lock'''
              with FileLock(UpperCamelCase ):
                     if os.path.exists(UpperCamelCase ) and not overwrite_cache:
                            logger.info(f"Loading features from cached file {cached_features_file}" )
                            lowercase__      			= torch.load(UpperCamelCase )
                     else:
                            logger.info(f"Creating features from dataset file at {data_dir}" )
                            lowercase__      			= token_classification_task.read_examples_from_file(UpperCamelCase   ,						UpperCamelCase )
                            # TODO clean up all this to leverage built-in features of tokenizers
                            lowercase__      			= token_classification_task.convert_examples_to_features(
                                UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						cls_token_at_end=bool(model_type in ['''xlnet'''] )   ,						cls_token=tokenizer.cls_token   ,						cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0   ,						sep_token=tokenizer.sep_token   ,						sep_token_extra=UpperCamelCase   ,						pad_on_left=bool(tokenizer.padding_side == '''left''' )   ,						pad_token=tokenizer.pad_token_id   ,						pad_token_segment_id=tokenizer.pad_token_type_id   ,						pad_token_label_id=self.pad_token_label_id   ,						)
                            logger.info(f"Saving features into cached file {cached_features_file}" )
                            torch.save(self.features   ,						UpperCamelCase )



       def __len__(self    :    Tuple ):

              '''simple docstring'''







              return len(self.features )






       def __getitem__(self    :    Tuple   ,						UpperCamelCase    :    Optional[int] ):

              '''simple docstring'''







              return self.features[i]


if is_tf_available():
 import tensorflow as tf
 class   __lowerCAmelCase						:





       '''simple docstring'''






       lowerCAmelCase__					:     List[InputFeatures]
       lowerCAmelCase__					:     int  					=							-100



       def __init__(self    :    Any   ,						UpperCamelCase    :    TokenClassificationTask   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    PreTrainedTokenizer   ,						UpperCamelCase    :    List[str]   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[int] = None   ,						UpperCamelCase    :    Optional[Any]=False   ,						UpperCamelCase    :    Split = Split.train   ,						):

              '''simple docstring'''







              lowercase__      			= token_classification_task.read_examples_from_file(UpperCamelCase   ,						UpperCamelCase )
              # TODO clean up all this to leverage built-in features of tokenizers
              lowercase__      			= token_classification_task.convert_examples_to_features(
                  UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase   ,						cls_token_at_end=bool(model_type in ['''xlnet'''] )   ,						cls_token=tokenizer.cls_token   ,						cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0   ,						sep_token=tokenizer.sep_token   ,						sep_token_extra=UpperCamelCase   ,						pad_on_left=bool(tokenizer.padding_side == '''left''' )   ,						pad_token=tokenizer.pad_token_id   ,						pad_token_segment_id=tokenizer.pad_token_type_id   ,						pad_token_label_id=self.pad_token_label_id   ,						)

              def gen():
                     for ex in self.features:
                            if ex.token_type_ids is None:
                                   yield (
                                       {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                                       ex.label_ids,
                                   )
                            else:
                                   yield (
                                       {
                                           "input_ids": ex.input_ids,
                                           "attention_mask": ex.attention_mask,
                                           "token_type_ids": ex.token_type_ids,
                                       },
                                       ex.label_ids,
                                   )

              if "token_type_ids" not in tokenizer.model_input_names:
                     lowercase__      			= tf.data.Dataset.from_generator(
                         UpperCamelCase   ,						({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa}, tf.intaa)   ,						(
                             {'''input_ids''': tf.TensorShape([None] ), '''attention_mask''': tf.TensorShape([None] )},
                             tf.TensorShape([None] ),
                         )   ,						)
              else:
                     lowercase__      			= tf.data.Dataset.from_generator(
                         UpperCamelCase   ,						({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa, '''token_type_ids''': tf.intaa}, tf.intaa)   ,						(
                             {
                                 '''input_ids''': tf.TensorShape([None] ),
                                 '''attention_mask''': tf.TensorShape([None] ),
                                 '''token_type_ids''': tf.TensorShape([None] ),
                             },
                             tf.TensorShape([None] ),
                         )   ,						)



       def       UpperCamelCase__		(self    :    Tuple ):

              '''simple docstring'''







              lowercase__      			= self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features ) ) )

              return self.dataset



       def __len__(self    :    List[Any] ):

              '''simple docstring'''







              return len(self.features )






       def __getitem__(self    :    int   ,						UpperCamelCase    :    Dict ):

              '''simple docstring'''







              return self.features[i]








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase							:   str  						=	{
    'configuration_rag': ['RagConfig'],
    'retrieval_rag': ['RagRetriever'],
    'tokenization_rag': ['RagTokenizer'],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Dict  						=	[
     'RagModel',
     'RagPreTrainedModel',
     'RagSequenceForGeneration',
     'RagTokenForGeneration',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase							:   Optional[int]  						=	[
     'TFRagModel',
     'TFRagPreTrainedModel',
     'TFRagSequenceForGeneration',
     'TFRagTokenForGeneration',
 ]


if TYPE_CHECKING:
 from .configuration_rag import RagConfig
 from .retrieval_rag import RagRetriever
 from .tokenization_rag import RagTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_rag import (
      TFRagModel,
      TFRagPreTrainedModel,
      TFRagSequenceForGeneration,
      TFRagTokenForGeneration,
  )

else:
 import sys

 lowerCamelCase							:   Tuple  						=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




