






"""simple docstring"""

import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
			from .tokenization_llama import LlamaTokenizer
else:
			lowercase           =							None

lowercase           =							logging.get_logger(__name__)
lowercase           =							{'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase      ,   lowercase           =							'''[INST]''', '''[/INST]'''
lowercase      ,   lowercase           =							'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
lowercase           =							'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''

class     A_				(   snake_case_	):
				UpperCAmelCase__										=					VOCAB_FILES_NAMES
				UpperCAmelCase__										=					LlamaTokenizer
				UpperCAmelCase__										=					'''left'''
				UpperCAmelCase__										=					['''input_ids''', '''attention_mask''']







				def __init__(				self				:    int    ,     __lowerCamelCase				:    Optional[Any]=None    ,     __lowerCamelCase				:    Union[str, Any]=None    ,     __lowerCamelCase				:    str=False    ,     __lowerCamelCase				:    List[Any]="<unk>"    ,     __lowerCamelCase				:    Optional[Any]="<s>"    ,     __lowerCamelCase				:    List[str]="</s>"    ,     __lowerCamelCase				:    Union[str, Any]=True    ,     __lowerCamelCase				:    Union[str, Any]=False    ,     **__lowerCamelCase				:    List[Any]    ,     )					->       Optional[int]:
										super().__init__(
										    vocab_file=__lowerCamelCase    ,     tokenizer_file=__lowerCamelCase    ,     clean_up_tokenization_spaces=__lowerCamelCase    ,     unk_token=__lowerCamelCase    ,     bos_token=__lowerCamelCase    ,     eos_token=__lowerCamelCase    ,     **__lowerCamelCase    ,     )
										__magic_name__      =  add_bos_token
										__magic_name__      =  add_eos_token
										self.update_post_processor()

										__magic_name__      =  vocab_file
										__magic_name__      =  False if not self.vocab_file else True







				def      _snake_case       (				self				:    str						)					->       List[str]:
										__magic_name__      =  self.bos_token
										__magic_name__      =  self.bos_token_id

										__magic_name__      =  self.eos_token
										__magic_name__      =  self.eos_token_id

										__magic_name__      =  f'''{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'''
										__magic_name__      =  f'''{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'''

										__magic_name__      =  []
										if self.add_bos_token:
																special_tokens.append((bos, bos_token_id)						)
										if self.add_eos_token:
																special_tokens.append((eos, eos_token_id)						)
										__magic_name__      =  processors.TemplateProcessing(
										    single=__lowerCamelCase    ,     pair=__lowerCamelCase    ,     special_tokens=__lowerCamelCase						)







				@property
				def      _snake_case       (				self				:    Any						)					->       Dict:
										return self._add_eos_token







				@property
				def      _snake_case       (				self				:    int						)					->       Union[str, Any]:
										return self._add_bos_token







				@add_eos_token.setter
				def      _snake_case       (				self				:    Optional[int]    ,     __lowerCamelCase				:    int						)					->       str:
										__magic_name__      =  value
										self.update_post_processor()







				@add_bos_token.setter
				def      _snake_case       (				self				:    Optional[Any]    ,     __lowerCamelCase				:    Optional[Any]						)					->       Tuple:
										__magic_name__      =  value
										self.update_post_processor()







				def      _snake_case       (				self				:    Tuple    ,     __lowerCamelCase				:    str    ,     __lowerCamelCase				:    Optional[str] = None						)					->       Tuple[str]:
										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer."						)

										if not os.path.isdir(__lowerCamelCase						):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																return
										__magic_name__      =  os.path.join(
										    __lowerCamelCase    ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

										if os.path.abspath(self.vocab_file						) != os.path.abspath(__lowerCamelCase						):
																copyfile(self.vocab_file    ,     __lowerCamelCase						)

										return (out_vocab_file,)







				def      _snake_case       (				self				:    Any    ,     __lowerCamelCase				:    "Conversation"						)					->       Union[str, Any]:
										__magic_name__      =  list(conversation.iter_texts()						)
										if not all([is_user for is_user, msg in dialogue[::2]]						) or not all(
										    [not is_user for is_user, msg in dialogue[1::2]]						):
																raise ValueError(
																    "The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)"						)

										__magic_name__      =  []
										if len(conversation.past_user_inputs						) > 0:
																if not conversation.past_user_inputs[0].startswith(__lowerCamelCase						) or E_SYS not in conversation.past_user_inputs[0]:
																						__magic_name__      =  (
																						    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
																						)
										elif not dialogue[0][1].startswith(__lowerCamelCase						) or E_SYS not in dialogue[0][1]:
																__magic_name__      =  (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

										dialog_tokens += sum(
										    [
										        [self.bos_token_id]
										        + self.encode(
										            f'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''    ,     add_special_tokens=__lowerCamelCase						)
										        + [self.eos_token_id]
										        for prompt, answer in zip(dialogue[::2]    ,     dialogue[1::2]						)
										    ]    ,     []    ,     )
										if not (dialogue[-1][0]):
																raise ValueError(f'''Last message must be from user, got {dialogue[-1]["role"]}'''						)
										dialog_tokens += [self.bos_token_id] + self.encode(
										    f'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''    ,     add_special_tokens=__lowerCamelCase						)
										return dialog_tokens











"""simple docstring"""

from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import ConvNextConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import TFConvNextForImageClassification, TFConvNextModel


if is_vision_available():
			from PIL import Image

			from transformers import ConvNextImageProcessor

class     A_				:







				def __init__(				self				:    Optional[int]    ,     __lowerCamelCase				:    Tuple    ,     __lowerCamelCase				:    int=1_3    ,     __lowerCamelCase				:    Any=3_2    ,     __lowerCamelCase				:    Dict=3    ,     __lowerCamelCase				:    List[Any]=4    ,     __lowerCamelCase				:    Union[str, Any]=[1_0, 2_0, 3_0, 4_0]    ,     __lowerCamelCase				:    int=[2, 2, 3, 2]    ,     __lowerCamelCase				:    str=True    ,     __lowerCamelCase				:    List[str]=True    ,     __lowerCamelCase				:    List[Any]=3_7    ,     __lowerCamelCase				:    Optional[Any]="gelu"    ,     __lowerCamelCase				:    str=1_0    ,     __lowerCamelCase				:    int=0.02    ,     __lowerCamelCase				:    Optional[int]=3    ,     __lowerCamelCase				:    Any=None    ,     )					->       List[Any]:
										__magic_name__      =  parent
										__magic_name__      =  batch_size
										__magic_name__      =  image_size
										__magic_name__      =  num_channels
										__magic_name__      =  num_stages
										__magic_name__      =  hidden_sizes
										__magic_name__      =  depths
										__magic_name__      =  is_training
										__magic_name__      =  use_labels
										__magic_name__      =  intermediate_size
										__magic_name__      =  hidden_act
										__magic_name__      =  type_sequence_label_size
										__magic_name__      =  initializer_range
										__magic_name__      =  scope







				def      _snake_case       (				self				:    Optional[Any]						)					->       int:
										__magic_name__      =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

										__magic_name__      =  None
										if self.use_labels:
																__magic_name__      =  ids_tensor([self.batch_size]    ,     self.type_sequence_label_size						)

										__magic_name__      =  self.get_config()

										return config, pixel_values, labels







				def      _snake_case       (				self				:    Optional[int]						)					->       Union[str, Any]:
										return ConvNextConfig(
										    num_channels=self.num_channels    ,     hidden_sizes=self.hidden_sizes    ,     depths=self.depths    ,     num_stages=self.num_stages    ,     hidden_act=self.hidden_act    ,     is_decoder=__lowerCamelCase    ,     initializer_range=self.initializer_range    ,     )







				def      _snake_case       (				self				:    List[Any]    ,     __lowerCamelCase				:    List[str]    ,     __lowerCamelCase				:    Union[str, Any]    ,     __lowerCamelCase				:    Optional[Any]						)					->       List[str]:
										__magic_name__      =  TFConvNextModel(config=__lowerCamelCase						)
										__magic_name__      =  model(__lowerCamelCase    ,     training=__lowerCamelCase						)
										# expected last hidden states: B, C, H // 32, W // 32
										self.parent.assertEqual(
										    result.last_hidden_state.shape    ,     (self.batch_size, self.hidden_sizes[-1], self.image_size // 3_2, self.image_size // 3_2)    ,     )







				def      _snake_case       (				self				:    List[Any]    ,     __lowerCamelCase				:    Union[str, Any]    ,     __lowerCamelCase				:    Tuple    ,     __lowerCamelCase				:    List[str]						)					->       int:
										__magic_name__      =  self.type_sequence_label_size
										__magic_name__      =  TFConvNextForImageClassification(__lowerCamelCase						)
										__magic_name__      =  model(__lowerCamelCase    ,     labels=__lowerCamelCase    ,     training=__lowerCamelCase						)
										self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.type_sequence_label_size)						)







				def      _snake_case       (				self				:    List[str]						)					->       str:
										__magic_name__      =  self.prepare_config_and_inputs()
										__magic_name__   ,     __magic_name__   ,     __magic_name__      =  config_and_inputs
										__magic_name__      =  {"pixel_values": pixel_values}
										return config, inputs_dict





@require_tf
class     A_				(   snake_case_       ,				snake_case_       ,				unittest.TestCase	):
				UpperCAmelCase__										=					(TFConvNextModel, TFConvNextForImageClassification) if is_tf_available() else ()
				UpperCAmelCase__										=					(
				    {'''feature-extraction''': TFConvNextModel, '''image-classification''': TFConvNextForImageClassification}
				    if is_tf_available()
				    else {}
				)

				UpperCAmelCase__										=					False
				UpperCAmelCase__										=					False
				UpperCAmelCase__										=					False
				UpperCAmelCase__										=					False
				UpperCAmelCase__										=					False







				def      _snake_case       (				self				:    str						)					->       str:
										__magic_name__      =  TFConvNextModelTester(self						)
										__magic_name__      =  ConfigTester(
										    self    ,     config_class=__lowerCamelCase    ,     has_text_modality=__lowerCamelCase    ,     hidden_size=3_7    ,     )







				@unittest.skip(reason="ConvNext does not use inputs_embeds"						)
				def      _snake_case       (				self				:    Dict						)					->       Dict:
										pass







				@unittest.skipIf(
				    not is_tf_available() or len(tf.config.list_physical_devices("GPU"						)						) == 0    ,     reason="TF does not support backprop for grouped convolutions on CPU."    ,     )
				@slow
				def      _snake_case       (				self				:    List[str]						)					->       Tuple:
										super().test_keras_fit()







				@unittest.skip(reason="ConvNext does not support input and output embeddings"						)
				def      _snake_case       (				self				:    str						)					->       Tuple:
										pass







				def      _snake_case       (				self				:    Union[str, Any]						)					->       Optional[int]:
										__magic_name__   ,     __magic_name__      =  self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																__magic_name__      =  model_class(__lowerCamelCase						)
																__magic_name__      =  inspect.signature(model.call						)
																# signature.parameters is an OrderedDict => so arg_names order is deterministic
																__magic_name__      =  [*signature.parameters.keys()]

																__magic_name__      =  ["pixel_values"]
																self.assertListEqual(arg_names[:1]    ,     __lowerCamelCase						)







				def      _snake_case       (				self				:    Dict						)					->       Optional[int]:
										__magic_name__      =  self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*__lowerCamelCase						)







				@unittest.skipIf(
				    not is_tf_available() or len(tf.config.list_physical_devices("GPU"						)						) == 0    ,     reason="TF does not support backprop for grouped convolutions on CPU."    ,     )
				def      _snake_case       (				self				:    Optional[int]						)					->       Tuple:
										super().test_dataset_conversion()







				def      _snake_case       (				self				:    int						)					->       List[str]:
										def check_hidden_states_output(__lowerCamelCase				:    Union[str, Any]    ,     __lowerCamelCase				:    Dict    ,     __lowerCamelCase				:    List[str]						):
																__magic_name__      =  model_class(__lowerCamelCase						)

																__magic_name__      =  model(**self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase						)						)
																__magic_name__      =  outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

																__magic_name__      =  self.model_tester.num_stages
																self.assertEqual(len(__lowerCamelCase						)    ,     expected_num_stages + 1						)

																# ConvNext's feature maps are of shape (batch_size, num_channels, height, width)
																self.assertListEqual(
																    list(hidden_states[0].shape[-2:]						)    ,     [self.model_tester.image_size // 4, self.model_tester.image_size // 4]    ,     )

										__magic_name__   ,     __magic_name__      =  self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
																__magic_name__      =  True
																check_hidden_states_output(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase						)

																# check that output_hidden_states also work using config
																del inputs_dict["output_hidden_states"]
																__magic_name__      =  True

																check_hidden_states_output(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase						)







				def      _snake_case       (				self				:    List[str]						)					->       Dict:
										__magic_name__   ,     __magic_name__      =  self.model_tester.prepare_config_and_inputs_for_common()

										def check_equivalence(__lowerCamelCase				:    Dict    ,     __lowerCamelCase				:    Any    ,     __lowerCamelCase				:    Union[str, Any]    ,     __lowerCamelCase				:    Any={}						):
																__magic_name__      =  model(__lowerCamelCase    ,     return_dict=__lowerCamelCase    ,     **__lowerCamelCase						)
																__magic_name__      =  model(__lowerCamelCase    ,     return_dict=__lowerCamelCase    ,     **__lowerCamelCase						).to_tuple()

																def recursive_check(__lowerCamelCase				:    Union[str, Any]    ,     __lowerCamelCase				:    Union[str, Any]						):
																						if isinstance(__lowerCamelCase    ,     (List, Tuple)						):
																												for tuple_iterable_value, dict_iterable_value in zip(__lowerCamelCase    ,     __lowerCamelCase						):
																																		recursive_check(__lowerCamelCase    ,     __lowerCamelCase						)
																						elif tuple_object is None:
																												return
																						else:
																												self.assertTrue(
																												    all(tf.equal(__lowerCamelCase    ,     __lowerCamelCase						)						)    ,     msg=(
																												        "Tuple and dict output are not equal. Difference:"
																												        f''' {tf.math.reduce_max(tf.abs(tuple_object - dict_object						)						)}'''
																												    )    ,     )

																						recursive_check(__lowerCamelCase    ,     __lowerCamelCase						)

										for model_class in self.all_model_classes:
																__magic_name__      =  model_class(__lowerCamelCase						)

																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase						)
																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase						)
																check_equivalence(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase						)

																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase    ,     return_labels=__lowerCamelCase						)
																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase    ,     return_labels=__lowerCamelCase						)
																check_equivalence(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase						)

																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase						)
																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase						)
																check_equivalence(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase    ,     {"output_hidden_states": True}						)

																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase    ,     return_labels=__lowerCamelCase						)
																__magic_name__      =  self._prepare_for_class(__lowerCamelCase    ,     __lowerCamelCase    ,     return_labels=__lowerCamelCase						)
																check_equivalence(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase    ,     {"output_hidden_states": True}						)







				def      _snake_case       (				self				:    Union[str, Any]						)					->       Dict:
										__magic_name__      =  self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_image_classification(*__lowerCamelCase						)







				@slow
				def      _snake_case       (				self				:    List[str]						)					->       Tuple:
										__magic_name__      =  TFConvNextModel.from_pretrained("facebook/convnext-tiny-224"						)
										self.assertIsNotNone(__lowerCamelCase						)






def _lowerCAmelCase						(	):

						'''simple docstring'''






						__magic_name__      =  Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png" )
						return image


@require_tf
@require_vision
class     A_				(   unittest.TestCase	):







				@cached_property
				def      _snake_case       (				self				:    Tuple						)					->       Any:
										return ConvNextImageProcessor.from_pretrained("facebook/convnext-tiny-224"						) if is_vision_available() else None







				@slow
				def      _snake_case       (				self				:    Any						)					->       str:
										__magic_name__      =  TFConvNextForImageClassification.from_pretrained("facebook/convnext-tiny-224"						)

										__magic_name__      =  self.default_image_processor
										__magic_name__      =  prepare_img()
										__magic_name__      =  image_processor(images=__lowerCamelCase    ,     return_tensors="tf"						)

										# forward pass
										__magic_name__      =  model(**__lowerCamelCase						)

										# verify the logits
										__magic_name__      =  tf.TensorShape((1, 1_0_0_0)						)
										self.assertEqual(outputs.logits.shape    ,     __lowerCamelCase						)

										__magic_name__      =  tf.constant([-0.0260, -0.4739, 0.1911]						)

										tf.debugging.assert_near(outputs.logits[0, :3]    ,     __lowerCamelCase    ,     atol=1e-4						)



