



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__A   			=logging.get_logger(__name__)


__A   			={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
__A   			={
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

__A   			={
    'facebook/bart-base': 1_0_2_4,
    'facebook/bart-large': 1_0_2_4,
    'facebook/bart-large-mnli': 1_0_2_4,
    'facebook/bart-large-cnn': 1_0_2_4,
    'facebook/bart-large-xsum': 1_0_2_4,
    'yjernite/bart_eli5': 1_0_2_4,
}

class 		UpperCAmelCase__			(			lowercase__   ):




			'''simple docstring'''






			UpperCamelCase									=       VOCAB_FILES_NAMES
			UpperCamelCase									=       PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase									=       ['''input_ids''', '''attention_mask''']
			UpperCamelCase									=       BartTokenizer




			def __init__(   self							: Union[str, Any]       ,       a_							: List[str]=None       ,       a_							: Dict=None       ,       a_							: Dict=None       ,       a_							: Dict="replace"       ,       a_							: List[Any]="<s>"       ,       a_							: str="</s>"       ,       a_							: Union[str, Any]="</s>"       ,       a_							: str="<s>"       ,       a_							: Optional[int]="<unk>"       ,       a_							: Dict="<pad>"       ,       a_							: int="<mask>"       ,       a_							: List[Any]=False       ,       a_							: Dict=True       ,       **a_							: int       ,       ):
					'''simple docstring'''





					super().__init__(
					    SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__       ,       tokenizer_file=SCREAMING_SNAKE_CASE__       ,       errors=SCREAMING_SNAKE_CASE__       ,       bos_token=SCREAMING_SNAKE_CASE__       ,       eos_token=SCREAMING_SNAKE_CASE__       ,       sep_token=SCREAMING_SNAKE_CASE__       ,       cls_token=SCREAMING_SNAKE_CASE__       ,       unk_token=SCREAMING_SNAKE_CASE__       ,       pad_token=SCREAMING_SNAKE_CASE__       ,       mask_token=SCREAMING_SNAKE_CASE__       ,       add_prefix_space=SCREAMING_SNAKE_CASE__       ,       trim_offsets=SCREAMING_SNAKE_CASE__       ,       **SCREAMING_SNAKE_CASE__       ,       )

					__UpperCAmelCase    :     Dict													=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
					if pre_tok_state.get('''add_prefix_space'''       ,       SCREAMING_SNAKE_CASE__ ) != add_prefix_space:
							__UpperCAmelCase    :     Any													=	getattr(SCREAMING_SNAKE_CASE__       ,       pre_tok_state.pop('''type''' ) )
							__UpperCAmelCase    :     Any													=	add_prefix_space
							__UpperCAmelCase    :     Optional[int]													=	pre_tok_class(**SCREAMING_SNAKE_CASE__ )

					__UpperCAmelCase    :     List[str]													=	add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					__UpperCAmelCase    :     int													=	'post_processor'
					__UpperCAmelCase    :     str													=	getattr(self.backend_tokenizer       ,       SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__ )
					if tokenizer_component_instance:
							__UpperCAmelCase    :     Dict													=	json.loads(tokenizer_component_instance.__getstate__() )

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
									__UpperCAmelCase    :     List[str]													=	tuple(state['''sep'''] )
							if "cls" in state:
									__UpperCAmelCase    :     Dict													=	tuple(state['''cls'''] )

							__UpperCAmelCase    :     List[str]													=	False

							if state.get('''add_prefix_space'''       ,       SCREAMING_SNAKE_CASE__ ) != add_prefix_space:
									__UpperCAmelCase    :     List[str]													=	add_prefix_space
									__UpperCAmelCase    :     str													=	True

							if state.get('''trim_offsets'''       ,       SCREAMING_SNAKE_CASE__ ) != trim_offsets:
									__UpperCAmelCase    :     Union[str, Any]													=	trim_offsets
									__UpperCAmelCase    :     Dict													=	True

							if changes_to_apply:
									__UpperCAmelCase    :     Any													=	getattr(SCREAMING_SNAKE_CASE__       ,       state.pop('''type''' ) )
									__UpperCAmelCase    :     Tuple													=	component_class(**SCREAMING_SNAKE_CASE__ )
									setattr(self.backend_tokenizer       ,       SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__ )




			@property
			def  snake_case__     (   self							: List[str] ):
					'''simple docstring'''





					if self._mask_token is None:
							if self.verbose:
									logger.error('''Using mask_token, but it is not set yet.''' )
							return None
					return str(self._mask_token )




			@mask_token.setter
			def  snake_case__     (   self							: List[Any]       ,       a_							: Any ):
					'''simple docstring'''





					__UpperCAmelCase    :     Tuple													=	AddedToken(SCREAMING_SNAKE_CASE__       ,       lstrip=SCREAMING_SNAKE_CASE__       ,       rstrip=SCREAMING_SNAKE_CASE__ ) if isinstance(SCREAMING_SNAKE_CASE__       ,       SCREAMING_SNAKE_CASE__ ) else value
					__UpperCAmelCase    :     List[Any]													=	value




			def  snake_case__     (   self							: int       ,       *a_							: Optional[int]       ,       **a_							: int ):
					'''simple docstring'''





					__UpperCAmelCase    :     Dict													=	kwargs.get('''is_split_into_words'''       ,       SCREAMING_SNAKE_CASE__ )

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
							    '''to use it with pretokenized inputs.''' )

					return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__       ,       **SCREAMING_SNAKE_CASE__ )




			def  snake_case__     (   self							: Dict       ,       *a_							: Dict       ,       **a_							: Optional[int] ):
					'''simple docstring'''





					__UpperCAmelCase    :     Union[str, Any]													=	kwargs.get('''is_split_into_words'''       ,       SCREAMING_SNAKE_CASE__ )

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
							    '''to use it with pretokenized inputs.''' )

					return super()._encode_plus(*SCREAMING_SNAKE_CASE__       ,       **SCREAMING_SNAKE_CASE__ )




			def  snake_case__     (   self							: List[Any]       ,       a_							: str       ,       a_							: Optional[str] = None ):
					'''simple docstring'''





					__UpperCAmelCase    :     List[Any]													=	self._tokenizer.model.save(SCREAMING_SNAKE_CASE__       ,       name=SCREAMING_SNAKE_CASE__ )
					return tuple(SCREAMING_SNAKE_CASE__ )




			def  snake_case__     (   self							: Optional[Any]       ,       a_							: Optional[int]       ,       a_							: Union[str, Any]=None ):
					'''simple docstring'''





					__UpperCAmelCase    :     Dict													=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




			def  snake_case__     (   self							: int       ,       a_							: List[int]       ,       a_							: Optional[List[int]] = None ):
					'''simple docstring'''





					__UpperCAmelCase    :     Optional[int]													=	[self.sep_token_id]
					__UpperCAmelCase    :     Union[str, Any]													=	[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]









import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
UpperCAmelCase_      : Dict  					=       {'LayoutLMv2Config', 'LayoutLMv3Config'}
@is_pipeline_test
class  SCREAMING_SNAKE_CASE__  (				unittest.TestCase			):
      snake_case__				:     List[str]        =						MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
      snake_case__				:     Optional[Any]        =						TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

      if model_mapping is not None:
            snake_case__				:     str        =						{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
      if tf_model_mapping is not None:
            snake_case__				:     List[Any]        =						{
                config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
            }
      @require_torch
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     )		->			Tuple:
             a_					:		List[Any]							=  pipeline(
                 task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='pt'     )

             a_					:		int							=  text_classifier('This is great !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}]     )

             a_					:		Tuple							=  text_classifier('This is great !'     ,							top_k=2     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}]     )

             a_					:		List[str]							=  text_classifier(['This is great !', 'This is bad']     ,							top_k=2     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[
                     [{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}],
                     [{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}],
                 ]     ,							)

             a_					:		Tuple							=  text_classifier('This is great !'     ,							top_k=1     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}]     )

             # Legacy behavior
             a_					:		Union[str, Any]							=  text_classifier('This is great !'     ,							return_all_scores=SCREAMING_SNAKE_CASE__     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}]     )

             a_					:		List[str]							=  text_classifier('This is great !'     ,							return_all_scores=SCREAMING_SNAKE_CASE__     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[[{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}]]     )

             a_					:		int							=  text_classifier(['This is great !', 'Something else']     ,							return_all_scores=SCREAMING_SNAKE_CASE__     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[
                     [{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}],
                     [{'label': 'LABEL_0', 'score': 0.504}, {'label': 'LABEL_1', 'score': 0.496}],
                 ]     ,							)

             a_					:		str							=  text_classifier(['This is great !', 'Something else']     ,							return_all_scores=SCREAMING_SNAKE_CASE__     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[
                     {'label': 'LABEL_0', 'score': 0.504},
                     {'label': 'LABEL_0', 'score': 0.504},
                 ]     ,							)
      @require_torch
      def        SCREAMING_SNAKE_CASE							(       self				:       int     )		->			Dict:
             import torch

             a_					:		List[Any]							=  pipeline(
                 task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='pt'     ,							device=torch.device('cpu'     )     ,							)

             a_					:		Any							=  text_classifier('This is great !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}]     )
      @require_tf
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			Optional[Any]:
             a_					:		List[str]							=  pipeline(
                 task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='tf'     )

             a_					:		Optional[int]							=  text_classifier('This is great !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'LABEL_0', 'score': 0.504}]     )
      @slow
      @require_torch
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			Optional[int]:
             a_					:		List[str]							=  pipeline('text-classification'     )

             a_					:		Dict							=  text_classifier('This is great !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'POSITIVE', 'score': 1.0}]     )
             a_					:		Union[str, Any]							=  text_classifier('This is bad !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'NEGATIVE', 'score': 1.0}]     )
             a_					:		Tuple							=  text_classifier('Birds are a type of animal'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'POSITIVE', 'score': 0.988}]     )
      @slow
      @require_tf
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     )		->			List[Any]:
             a_					:		Dict							=  pipeline('text-classification'     ,							framework='tf'     )

             a_					:		Optional[Any]							=  text_classifier('This is great !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'POSITIVE', 'score': 1.0}]     )
             a_					:		int							=  text_classifier('This is bad !'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'NEGATIVE', 'score': 1.0}]     )
             a_					:		Optional[int]							=  text_classifier('Birds are a type of animal'     )
             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': 'POSITIVE', 'score': 0.988}]     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     ,							SCREAMING_SNAKE_CASE__				:       Any     ,							SCREAMING_SNAKE_CASE__				:       int     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]     )		->			Any:
             a_					:		Optional[Any]							=  TextClassificationPipeline(model=SCREAMING_SNAKE_CASE__     ,							tokenizer=SCREAMING_SNAKE_CASE__     )
             return text_classifier, ["HuggingFace is in", "This is another test"]

      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     ,							SCREAMING_SNAKE_CASE__				:       Any     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]     )		->			Union[str, Any]:
             a_					:		List[str]							=  text_classifier.model
             # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
             a_					:		Union[str, Any]							=  'HuggingFace is in'
             a_					:		int							=  text_classifier(SCREAMING_SNAKE_CASE__     )

             self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}]     )
             self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()     )

             a_					:		Union[str, Any]							=  ['HuggingFace is in ', 'Paris is in France']
             a_					:		int							=  text_classifier(SCREAMING_SNAKE_CASE__     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}, {'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}]     ,							)
             self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()     )
             self.assertTrue(outputs[1]['label'] in model.config.idalabel.values()     )

             # Forcing to get all results with `top_k=None`
             # This is NOT the legacy format
             a_					:		List[Any]							=  text_classifier(SCREAMING_SNAKE_CASE__     ,							top_k=SCREAMING_SNAKE_CASE__     )
             a_					:		Dict							=  len(model.config.idalabel.values()     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[[{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}] * N, [{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}] * N]     ,							)

             a_					:		int							=  {'text': 'HuggingFace is in ', 'text_pair': 'Paris is in France'}
             a_					:		Optional[int]							=  text_classifier(SCREAMING_SNAKE_CASE__     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}     ,							)
             self.assertTrue(outputs['label'] in model.config.idalabel.values()     )

             # This might be used a text pair, but tokenizer + pipe interaction
             # makes it hard to understand that it's not using the pair properly
             # https://github.com/huggingface/transformers/issues/17305
             # We disabled this usage instead as it was outputting wrong outputs.
             a_					:		Any							=  [['HuggingFace is in ', 'Paris is in France']]
             with self.assertRaises(SCREAMING_SNAKE_CASE__     ):
                    text_classifier(SCREAMING_SNAKE_CASE__     )

             # This used to be valid for doing text pairs
             # We're keeping it working because of backward compatibility
             a_					:		Tuple							=  text_classifier([[['HuggingFace is in ', 'Paris is in France']]]     )
             self.assertEqual(
                 nested_simplify(SCREAMING_SNAKE_CASE__     )     ,							[{'label': ANY(SCREAMING_SNAKE_CASE__     ), 'score': ANY(SCREAMING_SNAKE_CASE__     )}]     ,							)
             self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()     )


