

'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


lowerCAmelCase_     =      logging.get_logger(__name__)

lowerCAmelCase_     =      {
    "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json",
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}







class lowerCamelCase   (    __lowerCAmelCase				):
				snake_case_							     = '''blenderbot'''
				snake_case_							     = ['''past_key_values''']
				snake_case_							     = {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}

				def __init__(       self,						lowercase_=8008,						lowercase_=128,						lowercase_=2,						lowercase_=10240,						lowercase_=32,						lowercase_=24,						lowercase_=10240,						lowercase_=32,						lowercase_=0.0,						lowercase_=0.0,						lowercase_=True,						lowercase_=True,						lowercase_="gelu",						lowercase_=2560,						lowercase_=0.1,						lowercase_=0.0,						lowercase_=0.0,						lowercase_=0.02,						lowercase_=1,						lowercase_=False,						lowercase_=0,						lowercase_=1,						lowercase_=2,						lowercase_=3,						lowercase_=2,						**lowercase_,						)    ->							Any:
					snake_case							=					vocab_size
					snake_case							=					max_position_embeddings
					snake_case							=					d_model
					snake_case							=					encoder_ffn_dim
					snake_case							=					encoder_layers
					snake_case							=					encoder_attention_heads
					snake_case							=					decoder_ffn_dim
					snake_case							=					decoder_layers
					snake_case							=					decoder_attention_heads
					snake_case							=					dropout
					snake_case							=					attention_dropout
					snake_case							=					activation_dropout
					snake_case							=					activation_function
					snake_case							=					init_std
					snake_case							=					encoder_layerdrop
					snake_case							=					decoder_layerdrop
					snake_case							=					use_cache
					snake_case							=					encoder_layers
					snake_case							=					scale_embedding  # scale factor will be sqrt(d_model) if True

					super().__init__(
					    pad_token_id=lowercase_,						bos_token_id=lowercase_,						eos_token_id=lowercase_,						is_encoder_decoder=lowercase_,						decoder_start_token_id=lowercase_,						encoder_no_repeat_ngram_size=lowercase_,						forced_eos_token_id=lowercase_,						**lowercase_,						)







class lowerCamelCase   (    __lowerCAmelCase				):

				@property
				def 					_lowerCamelCase    (       self )    ->							Mapping[str, Mapping[int, str]]:
					if self.task in ["default", "seq2seq-lm"]:
						snake_case							=					OrderedDict(
						    [
						        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
						        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
						    ] )
						if self.use_past:
							snake_case							=					{0: 'batch'}
							snake_case							=					{0: 'batch', 1: 'past_decoder_sequence + sequence'}
						else:
							snake_case							=					{0: 'batch', 1: 'decoder_sequence'}
							snake_case							=					{0: 'batch', 1: 'decoder_sequence'}
						if self.use_past:
							self.fill_with_past_key_values_(lowercase_,						direction='inputs' )
					elif self.task == "causal-lm":
						snake_case							=					OrderedDict(
						    [
						        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
						        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
						    ] )
						if self.use_past:
							snake_case			,			snake_case							=					self.num_layers
							for i in range(lowercase_ ):
								snake_case							=					{0: 'batch', 2: 'past_sequence + sequence'}
								snake_case							=					{0: 'batch', 2: 'past_sequence + sequence'}
					else:
						snake_case							=					OrderedDict(
						    [
						        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
						        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
						        ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
						        ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
						    ] )

					return common_inputs

				@property
				# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
				def 					_lowerCamelCase    (       self )    ->							Mapping[str, Mapping[int, str]]:
					if self.task in ["default", "seq2seq-lm"]:
						snake_case							=					super().outputs
					else:
						snake_case							=					super(lowercase_,						self ).outputs
						if self.use_past:
							snake_case			,			snake_case							=					self.num_layers
							for i in range(lowercase_ ):
								snake_case							=					{0: 'batch', 2: 'past_sequence + sequence'}
								snake_case							=					{0: 'batch', 2: 'past_sequence + sequence'}
					return common_outputs

				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_ = -1,						lowercase_ = -1,						lowercase_ = False,						lowercase_ = None,						)    ->							Mapping[str, Any]:
					snake_case							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    lowercase_,						lowercase_,						lowercase_,						lowercase_,						lowercase_ )
					# Generate decoder inputs
					snake_case							=					seq_length if not self.use_past else 1
					snake_case							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    lowercase_,						lowercase_,						lowercase_,						lowercase_,						lowercase_ )
					snake_case							=					{F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
					snake_case							=					dict(**lowercase_,						**lowercase_ )

					if self.use_past:
						if not is_torch_available():
							raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.' )
						else:
							import torch
						snake_case			,			snake_case							=					common_inputs['input_ids'].shape
						snake_case							=					common_inputs['decoder_input_ids'].shape[1]
						snake_case			,			snake_case							=					self.num_attention_heads
						snake_case							=					(
						    batch,
						    num_encoder_attention_heads,
						    encoder_seq_length,
						    self._config.hidden_size // num_encoder_attention_heads,
						)
						snake_case							=					decoder_seq_length
						snake_case							=					(
						    batch,
						    num_decoder_attention_heads,
						    decoder_past_length,
						    self._config.hidden_size // num_decoder_attention_heads,
						)
						snake_case							=					torch.cat(
						    [common_inputs['decoder_attention_mask'], torch.ones(lowercase_,						lowercase_ )],						dim=1 )
						snake_case							=					[]
						snake_case			,			snake_case							=					self.num_layers

						for _ in range(lowercase_ ):
							common_inputs["past_key_values"].append(
							    (
							        torch.zeros(lowercase_ ),
							        torch.zeros(lowercase_ ),
							        torch.zeros(lowercase_ ),
							        torch.zeros(lowercase_ ),
							    ) )
					return common_inputs

				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_ = -1,						lowercase_ = -1,						lowercase_ = False,						lowercase_ = None,						)    ->							Mapping[str, Any]:
					snake_case							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
					    lowercase_,						lowercase_,						lowercase_,						lowercase_,						lowercase_ )

					if self.use_past:
						if not is_torch_available():
							raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.' )
						else:
							import torch
						snake_case			,			snake_case							=					common_inputs['input_ids'].shape
						snake_case							=					seqlen
						snake_case			,			snake_case							=					self.num_layers
						snake_case			,			snake_case							=					self.num_attention_heads
						snake_case							=					(
						    batch,
						    num_encoder_attention_heads,
						    past_key_values_length,
						    self._config.hidden_size // num_encoder_attention_heads,
						)
						snake_case							=					common_inputs['attention_mask'].dtype
						snake_case							=					torch.cat(
						    [common_inputs['attention_mask'], torch.ones(lowercase_,						lowercase_,						dtype=lowercase_ )],						dim=1 )
						snake_case							=					[
						    (torch.zeros(lowercase_ ), torch.zeros(lowercase_ )) for _ in range(lowercase_ )
						]
					return common_inputs

				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_ = -1,						lowercase_ = -1,						lowercase_ = False,						lowercase_ = None,						)    ->							Mapping[str, Any]:
					# Copied from OnnxConfig.generate_dummy_inputs
					# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
					# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
					snake_case							=					compute_effective_axis_dimension(
					    lowercase_,						fixed_dimension=OnnxConfig.default_fixed_batch,						num_token_to_add=0 )

					# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
					snake_case							=					tokenizer.num_special_tokens_to_add(lowercase_ )
					snake_case							=					compute_effective_axis_dimension(
					    lowercase_,						fixed_dimension=OnnxConfig.default_fixed_sequence,						num_token_to_add=lowercase_ )

					# Generate dummy inputs according to compute batch and sequence
					snake_case							=					[' '.join([tokenizer.unk_token] ) * seq_length] * batch_size
					snake_case							=					dict(tokenizer(lowercase_,						return_tensors=lowercase_ ) )
					return common_inputs

				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_ = -1,						lowercase_ = -1,						lowercase_ = False,						lowercase_ = None,						)    ->							Mapping[str, Any]:
					if self.task in ["default", "seq2seq-lm"]:
						snake_case							=					self._generate_dummy_inputs_for_default_and_seqaseq_lm(
						    lowercase_,						batch_size=lowercase_,						seq_length=lowercase_,						is_pair=lowercase_,						framework=lowercase_ )

					elif self.task == "causal-lm":
						snake_case							=					self._generate_dummy_inputs_for_causal_lm(
						    lowercase_,						batch_size=lowercase_,						seq_length=lowercase_,						is_pair=lowercase_,						framework=lowercase_ )
					else:
						snake_case							=					self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
						    lowercase_,						batch_size=lowercase_,						seq_length=lowercase_,						is_pair=lowercase_,						framework=lowercase_ )

					return common_inputs

				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_,						lowercase_,						lowercase_ )    ->							List[str]:
					if self.task in ["default", "seq2seq-lm"]:
						snake_case							=					super()._flatten_past_key_values_(lowercase_,						lowercase_,						lowercase_,						lowercase_ )
					else:
						snake_case							=					super(lowercase_,						self )._flatten_past_key_values_(
						    lowercase_,						lowercase_,						lowercase_,						lowercase_ )






				def 					_lowerCamelCase    (       self,						lowercase_,						lowercase_ )    ->							List[str]:
					if direction not in ["inputs", "outputs"]:
						raise ValueError(F'''direction must either be "inputs" or "outputs", but {direction} was given''' )

					snake_case							=					'past_key_values' if direction == 'inputs' else 'present'
					snake_case			,			snake_case							=					self.num_layers

					snake_case							=					'past_encoder_sequence'
					snake_case							=					'past_decoder_sequence' if direction == 'inputs' else 'past_decoder_sequence + sequence'

					for i in range(lowercase_ ):
						snake_case							=					{0: 'batch', 2: decoder_sequence}
						snake_case							=					{0: 'batch', 2: decoder_sequence}
						snake_case							=					{0: 'batch', 2: encoder_sequence}
						snake_case							=					{0: 'batch', 2: encoder_sequence}












'''simple docstring'''






import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				_A :				int				=  tmp_path / '''text.txt'''
				_A :				int				=  textwrap.dedent(
				    '''\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        ''')
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Tuple):



				"""simple docstring"""


				_A :				str				=  tmp_path / '''text_with_image.txt'''
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.mark.parametrize('''keep_linebreaks'''   , [True, False])
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read().splitlines(keepends=lowerCAmelCase)
				_A :				Any				=  Text(keep_linebreaks=lowerCAmelCase   , encoding='''utf-8''')
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				Dict				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content





@require_pil
def      lowercase  (  lowerCAmelCase :						Dict):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Tuple				=  f.read().splitlines()[0]
				_A :				Optional[int]				=  Text(encoding='''utf-8'''   , features=Features({'''image''': Image()}))
				_A :				Any				=  text._generate_tables([[text_file_with_image]])
				_A :				str				=  pa.concat_tables([table for _, table in generator])
				assert pa_table.schema.field('''image''').type == Image()()
				_A :				List[str]				=  pa_table.to_pydict()['''image''']
				assert generated_content == [{"path": image_file, "bytes": None}]





@pytest.mark.parametrize('''sample_by'''   , ['''line''', '''paragraph''', '''document'''])
def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						Optional[int]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read()
				if sample_by == "line":
								_A :				str				=  expected_content.splitlines()
				elif sample_by == "paragraph":
								_A :				str				=  expected_content.split('''\n\n''')
				elif sample_by == "document":
								_A :				Tuple				=  [expected_content]
				_A :				str				=  Text(sample_by=lowerCAmelCase   , encoding='''utf-8'''   , chunksize=100)
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				int				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content
