


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__a       =					logging.get_logger(__name__)


__a       =					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
__a       =					{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

__a       =					{
    'facebook/bart-base': 10_24,
    'facebook/bart-large': 10_24,
    'facebook/bart-large-mnli': 10_24,
    'facebook/bart-large-cnn': 10_24,
    'facebook/bart-large-xsum': 10_24,
    'yjernite/bart_eli5': 10_24,
}







class  __SCREAMING_SNAKE_CASE      (	UpperCamelCase__      ):
      A :							Dict										=   VOCAB_FILES_NAMES
      A :							Any										=   PRETRAINED_VOCAB_FILES_MAP
      A :							Union[str, Any]										=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A :							int										=   ["""input_ids""", """attention_mask"""]
      A :							str										=   BartTokenizer


      def __init__(							self							,  SCREAMING_SNAKE_CASE__=None							,  SCREAMING_SNAKE_CASE__=None							,  SCREAMING_SNAKE_CASE__=None							,  SCREAMING_SNAKE_CASE__="replace"							,  SCREAMING_SNAKE_CASE__="<s>"							,  SCREAMING_SNAKE_CASE__="</s>"							,  SCREAMING_SNAKE_CASE__="</s>"							,  SCREAMING_SNAKE_CASE__="<s>"							,  SCREAMING_SNAKE_CASE__="<unk>"							,  SCREAMING_SNAKE_CASE__="<pad>"							,  SCREAMING_SNAKE_CASE__="<mask>"							,  SCREAMING_SNAKE_CASE__=False							,  SCREAMING_SNAKE_CASE__=True							,  **SCREAMING_SNAKE_CASE__							,  ):
            super().__init__(
                __a							,  __a							,  tokenizer_file=__a							,  errors=__a							,  bos_token=__a							,  eos_token=__a							,  sep_token=__a							,  cls_token=__a							,  unk_token=__a							,  pad_token=__a							,  mask_token=__a							,  add_prefix_space=__a							,  trim_offsets=__a							,  **__a							,  )

            lowercase	:      str          =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
            if pre_tok_state.get('''add_prefix_space'''							,  __a						) != add_prefix_space:
                  lowercase	:      Any          =						getattr(__a							,  pre_tok_state.pop('''type'''						)						)
                  lowercase	:      Dict          =						add_prefix_space
                  lowercase	:      Any          =						pre_tok_class(**__a						)

            lowercase	:      Optional[Any]          =						add_prefix_space

            # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
            lowercase	:      Tuple          =						'''post_processor'''
            lowercase	:      Union[str, Any]          =						getattr(self.backend_tokenizer							,  __a							,  __a						)
            if tokenizer_component_instance:
                  lowercase	:      Any          =						json.loads(tokenizer_component_instance.__getstate__()						)

                  # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                  if "sep" in state:
                        lowercase	:      Optional[int]          =						tuple(state['''sep''']						)
                  if "cls" in state:
                        lowercase	:      str          =						tuple(state['''cls''']						)

                  lowercase	:      Optional[Any]          =						False

                  if state.get('''add_prefix_space'''							,  __a						) != add_prefix_space:
                        lowercase	:      int          =						add_prefix_space
                        lowercase	:      Dict          =						True

                  if state.get('''trim_offsets'''							,  __a						) != trim_offsets:
                        lowercase	:      Tuple          =						trim_offsets
                        lowercase	:      Dict          =						True

                  if changes_to_apply:
                        lowercase	:      int          =						getattr(__a							,  state.pop('''type'''						)						)
                        lowercase	:      str          =						component_class(**__a						)
                        setattr(self.backend_tokenizer							,  __a							,  __a						)


      @property
      def 				__lowerCamelCase	(							self						):
            if self._mask_token is None:
                  if self.verbose:
                        logger.error('''Using mask_token, but it is not set yet.'''						)
                  return None
            return str(self._mask_token						)


      @mask_token.setter
      def 				__lowerCamelCase	(							self							,  SCREAMING_SNAKE_CASE__						):
            lowercase	:      Dict          =						AddedToken(__a							,  lstrip=__a							,  rstrip=__a						) if isinstance(__a							,  __a						) else value
            lowercase	:      Tuple          =						value


      def 				__lowerCamelCase	(							self							,  *SCREAMING_SNAKE_CASE__							,  **SCREAMING_SNAKE_CASE__						):
            lowercase	:      str          =						kwargs.get('''is_split_into_words'''							,  __a						)

            if is_split_into_words and not self.add_prefix_space:
                  raise ValueError(
                      f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                      '''to use it with pretokenized inputs.'''						)

            return super()._batch_encode_plus(*__a							,  **__a						)


      def 				__lowerCamelCase	(							self							,  *SCREAMING_SNAKE_CASE__							,  **SCREAMING_SNAKE_CASE__						):
            lowercase	:      Optional[Any]          =						kwargs.get('''is_split_into_words'''							,  __a						)

            if is_split_into_words and not self.add_prefix_space:
                  raise ValueError(
                      f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                      '''to use it with pretokenized inputs.'''						)

            return super()._encode_plus(*__a							,  **__a						)


      def 				__lowerCamelCase	(							self							,  SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__ = None						):
            lowercase	:      List[Any]          =						self._tokenizer.model.save(__a							,  name=__a						)
            return tuple(__a						)


      def 				__lowerCamelCase	(							self							,  SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__=None						):
            lowercase	:      List[str]          =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
                  return output

            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






      def 				__lowerCamelCase	(							self							,  SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__ = None						):
            lowercase	:      int          =						[self.sep_token_id]
            lowercase	:      List[str]          =						[self.cls_token_id]

            if token_ids_a is None:
                  return len(cls + token_ids_a + sep						) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]


'''simple docstring'''





import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_:     str  =logging.getLogger(__name__)




@dataclass(frozen=UpperCamelCase__	)
class        __A      :
    a__       :						str
    a__       :						str
    a__       :						Optional[str]          =  None
    a__       :						Optional[str]          =  None
    a__       :						Optional[str]          =  None




@dataclass(frozen=UpperCamelCase__	)
class        __A      :
    a__       :						List[int]
    a__       :						Optional[List[int]]          =  None
    a__       :						Optional[List[int]]          =  None
    a__       :						Optional[Union[int, float]]          =  None
    a__       :						Optional[int]          =  None


if is_torch_available():
    import torch
    from torch.utils.data import Dataset




    class        __A      ( UpperCamelCase__	):
        a__       :						List[InputFeatures]



        def __init__(self    :    Any   ,			__a    :    str   ,			__a    :    PreTrainedTokenizer   ,			__a    :    str   ,			__a    :    Optional[int] = None   ,			__a    :    Dict=False   ,			__a    :    bool = False   ,			):
             UpperCAmelCase_           =   hans_processors[task]()

             UpperCAmelCase_           =   os.path.join(
                 __a   ,			"cached_{}_{}_{}_{}".format(
                     "dev" if evaluate else "train"   ,			tokenizer.__class__.__name__   ,			str(__a )   ,			__a   ,			)   ,			)
             UpperCAmelCase_           =   processor.get_labels()
             if tokenizer.__class__ in (
                 RobertaTokenizer,
                 RobertaTokenizerFast,
                 XLMRobertaTokenizer,
                 BartTokenizer,
                 BartTokenizerFast,
             ):
                  # HACK(label indices are swapped in RoBERTa pretrained model)
                  UpperCAmelCase_    , UpperCAmelCase_           =   label_list[2], label_list[1]
             UpperCAmelCase_           =   label_list

             # Make sure only the first process in distributed training processes the dataset,
             # and the others will use the cache.
             UpperCAmelCase_           =   cached_features_file + ".lock"
             with FileLock(__a ):
                  if os.path.exists(__a ) and not overwrite_cache:
                       logger.info(f"""Loading features from cached file {cached_features_file}""" )
                       UpperCAmelCase_           =   torch.load(__a )
                  else:
                       logger.info(f"""Creating features from dataset file at {data_dir}""" )

                       UpperCAmelCase_           =   (
                           processor.get_dev_examples(__a ) if evaluate else processor.get_train_examples(__a )
                       )

                       logger.info("Training examples: %s"   ,			len(__a ) )
                       UpperCAmelCase_           =   hans_convert_examples_to_features(__a   ,			__a   ,			__a   ,			__a )
                       logger.info("Saving features into cached file %s"   ,			__a )
                       torch.save(self.features   ,			__a )



        def __len__(self    :    List[Any] ):
             return len(self.features )



        def __getitem__(self    :    Any   ,			__a    :    Optional[Any] ):
             return self.features[i]






        def 					_lowercase						(self    :    Union[str, Any] ):
             return self.label_list


if is_tf_available():
    import tensorflow as tf




    class        __A      :
        a__       :						List[InputFeatures]



        def __init__(self    :    Union[str, Any]   ,			__a    :    str   ,			__a    :    PreTrainedTokenizer   ,			__a    :    str   ,			__a    :    Optional[int] = 128   ,			__a    :    Any=False   ,			__a    :    bool = False   ,			):
             UpperCAmelCase_           =   hans_processors[task]()
             UpperCAmelCase_           =   processor.get_labels()
             if tokenizer.__class__ in (
                 RobertaTokenizer,
                 RobertaTokenizerFast,
                 XLMRobertaTokenizer,
                 BartTokenizer,
                 BartTokenizerFast,
             ):
                  # HACK(label indices are swapped in RoBERTa pretrained model)
                  UpperCAmelCase_    , UpperCAmelCase_           =   label_list[2], label_list[1]
             UpperCAmelCase_           =   label_list

             UpperCAmelCase_           =   processor.get_dev_examples(__a ) if evaluate else processor.get_train_examples(__a )
             UpperCAmelCase_           =   hans_convert_examples_to_features(__a   ,			__a   ,			__a   ,			__a )

             def gen():
                  for ex_index, ex in tqdm.tqdm(enumerate(self.features )   ,			desc="convert examples to features" ):
                       if ex_index % 10000 == 0:
                            logger.info("Writing example %d of %d" % (ex_index, len(__a )) )

                       yield (
                           {
                               "example_id": 0,
                               "input_ids": ex.input_ids,
                               "attention_mask": ex.attention_mask,
                               "token_type_ids": ex.token_type_ids,
                           },
                           ex.label,
                       )

             UpperCAmelCase_           =   tf.data.Dataset.from_generator(
                 __a   ,			(
                     {
                         "example_id": tf.intaa,
                         "input_ids": tf.intaa,
                         "attention_mask": tf.intaa,
                         "token_type_ids": tf.intaa,
                     },
                     tf.intaa,
                 )   ,			(
                     {
                         "example_id": tf.TensorShape([] ),
                         "input_ids": tf.TensorShape([None, None] ),
                         "attention_mask": tf.TensorShape([None, None] ),
                         "token_type_ids": tf.TensorShape([None, None] ),
                     },
                     tf.TensorShape([] ),
                 )   ,			)



        def 					_lowercase						(self    :    int ):
             return self.dataset



        def __len__(self    :    Any ):
             return len(self.features )



        def __getitem__(self    :    int   ,			__a    :    Union[str, Any] ):
             return self.features[i]






        def 					_lowercase						(self    :    int ):
             return self.label_list




class        __A      ( UpperCamelCase__	):



    def 					_lowercase						(self    :    List[Any]   ,			__a    :    Dict ):
         return self._create_examples(self._read_tsv(os.path.join(__a   ,			"heuristics_train_set.txt" ) )   ,			"train" )



    def 					_lowercase						(self    :    Any   ,			__a    :    List[Any] ):
         return self._create_examples(self._read_tsv(os.path.join(__a   ,			"heuristics_evaluation_set.txt" ) )   ,			"dev" )



    def 					_lowercase						(self    :    Any ):
         return ["contradiction", "entailment", "neutral"]






    def 					_lowercase						(self    :    Union[str, Any]   ,			__a    :    Optional[int]   ,			__a    :    Union[str, Any] ):
         UpperCAmelCase_           =   []
         for i, line in enumerate(__a ):
              if i == 0:
                   continue
              UpperCAmelCase_           =   "%s-%s" % (set_type, line[0])
              UpperCAmelCase_           =   line[5]
              UpperCAmelCase_           =   line[6]
              UpperCAmelCase_           =   line[7][2:] if line[7].startswith("ex" ) else line[7]
              UpperCAmelCase_           =   line[0]
              examples.append(InputExample(guid=__a   ,			text_a=__a   ,			text_b=__a   ,			label=__a   ,			pairID=__a ) )
         return examples





def 	lowerCAmelCase_				(					snake_case_       : List[InputExample]					,    snake_case_       : List[str]					,    snake_case_       : int					,    snake_case_       : PreTrainedTokenizer					,    )    ->      Optional[Any]:


     '''simple docstring'''




     UpperCAmelCase_           =   {label: i for i, label in enumerate(snake_case_			)}

     UpperCAmelCase_           =   []
     for ex_index, example in tqdm.tqdm(enumerate(snake_case_			)					,    desc="convert examples to features"			):
          if ex_index % 1_00_00 == 0:
               logger.info("Writing example %d" % (ex_index)			)

          UpperCAmelCase_           =   tokenizer(
              example.text_a					,    example.text_b					,    add_special_tokens=snake_case_					,    max_length=snake_case_					,    padding="max_length"					,    truncation=snake_case_					,    return_overflowing_tokens=snake_case_					,    )

          UpperCAmelCase_           =   label_map[example.label] if example.label in label_map else 0

          UpperCAmelCase_           =   int(example.pairID			)

          features.append(InputFeatures(**snake_case_					,    label=snake_case_					,    pairID=snake_case_			)			)

     for i, example in enumerate(examples[:5]			):
          logger.info("*** Example ***"			)
          logger.info(f"""guid: {example}"""			)
          logger.info(f"""features: {features[i]}"""			)

     return features


SCREAMING_SNAKE_CASE_:     int  ={
    'hans': 3,
}






SCREAMING_SNAKE_CASE_:     Any  ={
    'hans': HansProcessor,
}




