
import csv
import os

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.csv import CsvDatasetReader, CsvDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def lowerCamelCase_       (			A			:      Union[str, Any]		,					A			:      Tuple						):




      """simple docstring"""
      assert isinstance(A		,					A						)
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''		,					[False, True]						)
def lowerCamelCase_       (			A			:      List[Any]		,					A			:      Any		,					A			:      Optional[Any]						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCAmelCase_						=      CsvDatasetReader(A		,					cache_dir=A		,					keep_in_memory=A						).read()
      _check_csv_dataset(A		,					A						)
@pytest.mark.parametrize(
    '''features'''		,					[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]		,					)
def lowerCamelCase_       (			A			:      Optional[int]		,					A			:      Tuple		,					A			:      List[str]						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      lowerCAmelCase_						=      features.copy() if features else default_expected_features
      lowerCAmelCase_						=      (
          Features({feature: Value(A						) for feature, dtype in features.items()}						) if features is not None else None
      )
      lowerCAmelCase_						=      CsvDatasetReader(A		,					features=A		,					cache_dir=A						).read()
      _check_csv_dataset(A		,					A						)
@pytest.mark.parametrize('''split'''		,					[None, NamedSplit('''train'''						), '''train''', '''test''']						)
def lowerCamelCase_       (			A			:      str		,					A			:      Union[str, Any]		,					A			:      str						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      lowerCAmelCase_						=      CsvDatasetReader(A		,					cache_dir=A		,					split=A						).read()
      _check_csv_dataset(A		,					A						)
      assert dataset.split == split if split else "train"
@pytest.mark.parametrize('''path_type'''		,					[str, list]						)
def lowerCamelCase_       (			A			:      Optional[Any]		,					A			:      Tuple		,					A			:      str						):




      """simple docstring"""
      if issubclass(A		,					A						):
            lowerCAmelCase_						=      csv_path
      elif issubclass(A		,					A						):
            lowerCAmelCase_						=      [csv_path]
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      lowerCAmelCase_						=      CsvDatasetReader(A		,					cache_dir=A						).read()
      _check_csv_dataset(A		,					A						)
def lowerCamelCase_       (			A			:      str		,					A			:      List[str]		,					A			:      List[str]=("train",)						):




      """simple docstring"""
      assert isinstance(A		,					A						)
      for split in splits:
            lowerCAmelCase_						=      dataset_dict[split]
            assert dataset.num_rows == 4
            assert dataset.num_columns == 3
            assert dataset.column_names == ["col_1", "col_2", "col_3"]
            for feature, expected_dtype in expected_features.items():
                  assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''		,					[False, True]						)
def lowerCamelCase_       (			A			:      Optional[Any]		,					A			:      Optional[int]		,					A			:      Optional[Any]						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCAmelCase_						=      CsvDatasetReader({'''train''': csv_path}		,					cache_dir=A		,					keep_in_memory=A						).read()
      _check_csv_datasetdict(A		,					A						)
@pytest.mark.parametrize(
    '''features'''		,					[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]		,					)
def lowerCamelCase_       (			A			:      str		,					A			:      Any		,					A			:      str						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      lowerCAmelCase_						=      features.copy() if features else default_expected_features
      lowerCAmelCase_						=      (
          Features({feature: Value(A						) for feature, dtype in features.items()}						) if features is not None else None
      )
      lowerCAmelCase_						=      CsvDatasetReader({'''train''': csv_path}		,					features=A		,					cache_dir=A						).read()
      _check_csv_datasetdict(A		,					A						)
@pytest.mark.parametrize('''split'''		,					[None, NamedSplit('''train'''						), '''train''', '''test''']						)
def lowerCamelCase_       (			A			:      Dict		,					A			:      str		,					A			:      Any						):




      """simple docstring"""
      if split:
            lowerCAmelCase_						=      {split: csv_path}
      else:
            lowerCAmelCase_						=      {'''train''': csv_path, '''test''': csv_path}
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      lowerCAmelCase_						=      CsvDatasetReader(A		,					cache_dir=A						).read()
      _check_csv_datasetdict(A		,					A		,					splits=list(path.keys()						)						)
      assert all(dataset[split].split == split for split in path.keys()						)
def lowerCamelCase_       (			A			:      Optional[int]						):




      """simple docstring"""
      with open(A		,					encoding='''utf-8'''						) as csvfile:
            yield from csv.reader(A						)
def lowerCamelCase_       (			A			:      Optional[Any]		,					A			:      Any						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      os.path.join(A		,					'''tmp.csv'''						)
      lowerCAmelCase_						=      CsvDatasetReader({'''train''': csv_path}		,					cache_dir=A						).read()
      CsvDatasetWriter(dataset['''train''']		,					A		,					num_proc=1						).write()

      lowerCAmelCase_						=      iter_csv_file(A						)
      lowerCAmelCase_						=      iter_csv_file(A						)

      for rowa, rowa in zip(A		,					A						):
            assert rowa == rowa
def lowerCamelCase_       (			A			:      Optional[int]		,					A			:      Dict						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      os.path.join(A		,					'''tmp.csv'''						)
      lowerCAmelCase_						=      CsvDatasetReader({'''train''': csv_path}		,					cache_dir=A						).read()
      CsvDatasetWriter(dataset['''train''']		,					A		,					num_proc=2						).write()

      lowerCAmelCase_						=      iter_csv_file(A						)
      lowerCAmelCase_						=      iter_csv_file(A						)

      for rowa, rowa in zip(A		,					A						):
            assert rowa == rowa




def lowerCamelCase_       (			A			:      List[str]		,					A			:      List[str]						):




      """simple docstring"""
      lowerCAmelCase_						=      tmp_path / '''cache'''
      lowerCAmelCase_						=      os.path.join(A		,					'''tmp.csv'''						)
      lowerCAmelCase_						=      CsvDatasetReader({'''train''': csv_path}		,					cache_dir=A						).read()
      with pytest.raises(A						):
            CsvDatasetWriter(dataset['''train''']		,					A		,					num_proc=0						)





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_snake_case		  =   logging.get_logger(__name__)

_snake_case		  =   {
    "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json",
    # See all MBART models at https://huggingface.co/models?filter=mbart
}


class 			UpperCamelCase_				(  A     ):




      '''simple docstring'''







      a		:Optional[int]						 =      'mbart'
      a		:List[Any]						 =      ['past_key_values']
      a		:Optional[Any]						 =      {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}







      def __init__( self					,					_UpperCAmelCase=50_265					,					_UpperCAmelCase=1_024					,					_UpperCAmelCase=12					,					_UpperCAmelCase=4_096					,					_UpperCAmelCase=16					,					_UpperCAmelCase=12					,					_UpperCAmelCase=4_096					,					_UpperCAmelCase=16					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=True					,					_UpperCAmelCase=True					,					_UpperCAmelCase="gelu"					,					_UpperCAmelCase=1_024					,					_UpperCAmelCase=0.1					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.02					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=False					,					_UpperCAmelCase=1					,					_UpperCAmelCase=0					,					_UpperCAmelCase=2					,					_UpperCAmelCase=2					,					**_UpperCAmelCase					,					):
            lowerCAmelCase_						=      vocab_size
            lowerCAmelCase_						=      max_position_embeddings
            lowerCAmelCase_						=      d_model
            lowerCAmelCase_						=      encoder_ffn_dim
            lowerCAmelCase_						=      encoder_layers
            lowerCAmelCase_						=      encoder_attention_heads
            lowerCAmelCase_						=      decoder_ffn_dim
            lowerCAmelCase_						=      decoder_layers
            lowerCAmelCase_						=      decoder_attention_heads
            lowerCAmelCase_						=      dropout
            lowerCAmelCase_						=      attention_dropout
            lowerCAmelCase_						=      activation_dropout
            lowerCAmelCase_						=      activation_function
            lowerCAmelCase_						=      init_std
            lowerCAmelCase_						=      encoder_layerdrop
            lowerCAmelCase_						=      decoder_layerdrop
            lowerCAmelCase_						=      classifier_dropout
            lowerCAmelCase_						=      use_cache
            lowerCAmelCase_						=      encoder_layers
            lowerCAmelCase_						=      scale_embedding  # scale factor will be sqrt(d_model) if True
            super().__init__(
                pad_token_id=_UpperCAmelCase					,					bos_token_id=_UpperCAmelCase					,					eos_token_id=_UpperCAmelCase					,					is_encoder_decoder=_UpperCAmelCase					,					forced_eos_token_id=_UpperCAmelCase					,					**_UpperCAmelCase					,					)


class 			UpperCamelCase_				(  A     ):




      '''simple docstring'''







      @property
      def lowercase__		( self):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                      ])

                  if self.use_past:
                        lowerCAmelCase_						=      {0: '''batch'''}
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
                  else:
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''decoder_sequence'''}
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''decoder_sequence'''}

                  if self.use_past:
                        self.fill_with_past_key_values_(_UpperCAmelCase					,					direction='''inputs''')
            elif self.task == "causal-lm":
                  # TODO: figure this case out.
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                      ])
                  if self.use_past:
                        lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                        for i in range(_UpperCAmelCase):
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
            else:
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                          ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                      ])

            return common_inputs







      @property
      def lowercase__		( self):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      super().outputs
            else:
                  lowerCAmelCase_						=      super(_UpperCAmelCase					,					self).outputs
                  if self.use_past:
                        lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                        for i in range(_UpperCAmelCase):
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
            return common_outputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)

            # Generate decoder inputs
            lowerCAmelCase_						=      seq_length if not self.use_past else 1
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)
            lowerCAmelCase_						=      {f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
            lowerCAmelCase_						=      dict(**_UpperCAmelCase					,					**_UpperCAmelCase)

            if self.use_past:
                  if not is_torch_available():
                        raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.''')
                  else:
                        import torch
                  lowerCAmelCase_			,		lowerCAmelCase_						=      common_inputs['''input_ids'''].shape
                  lowerCAmelCase_						=      common_inputs['''decoder_input_ids'''].shape[1]
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_attention_heads
                  lowerCAmelCase_						=      (
                      batch,
                      num_encoder_attention_heads,
                      encoder_seq_length,
                      self._config.hidden_size // num_encoder_attention_heads,
                  )
                  lowerCAmelCase_						=      decoder_seq_length + 3
                  lowerCAmelCase_						=      (
                      batch,
                      num_decoder_attention_heads,
                      decoder_past_length,
                      self._config.hidden_size // num_decoder_attention_heads,
                  )

                  lowerCAmelCase_						=      torch.cat(
                      [common_inputs['''decoder_attention_mask'''], torch.ones(_UpperCAmelCase					,					_UpperCAmelCase)]					,					dim=1)

                  lowerCAmelCase_						=      []
                  # If the number of encoder and decoder layers are present in the model configuration, both are considered
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                  lowerCAmelCase_						=      min(_UpperCAmelCase					,					_UpperCAmelCase)
                  lowerCAmelCase_						=      max(_UpperCAmelCase					,					_UpperCAmelCase) - min_num_layers
                  lowerCAmelCase_						=      '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

                  for _ in range(_UpperCAmelCase):
                        common_inputs["past_key_values"].append(
                            (
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                            ))
                  # TODO: test this.
                  lowerCAmelCase_						=      encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
                  for _ in range(_UpperCAmelCase					,					_UpperCAmelCase):
                        common_inputs["past_key_values"].append((torch.zeros(_UpperCAmelCase), torch.zeros(_UpperCAmelCase)))
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)

            if self.use_past:
                  if not is_torch_available():
                        raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.''')
                  else:
                        import torch
                  lowerCAmelCase_			,		lowerCAmelCase_						=      common_inputs['''input_ids'''].shape
                  # Not using the same length for past_key_values
                  lowerCAmelCase_						=      seqlen + 2
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_attention_heads
                  lowerCAmelCase_						=      (
                      batch,
                      num_encoder_attention_heads,
                      past_key_values_length,
                      self._config.hidden_size // num_encoder_attention_heads,
                  )

                  lowerCAmelCase_						=      common_inputs['''attention_mask'''].dtype
                  lowerCAmelCase_						=      torch.cat(
                      [common_inputs['''attention_mask'''], torch.ones(_UpperCAmelCase					,					_UpperCAmelCase					,					dtype=_UpperCAmelCase)]					,					dim=1)
                  lowerCAmelCase_						=      [
                      (torch.zeros(_UpperCAmelCase), torch.zeros(_UpperCAmelCase)) for _ in range(_UpperCAmelCase)
                  ]
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            # Copied from OnnxConfig.generate_dummy_inputs
            # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
            # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
            lowerCAmelCase_						=      compute_effective_axis_dimension(
                _UpperCAmelCase					,					fixed_dimension=OnnxConfig.default_fixed_batch					,					num_token_to_add=0)

            # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
            lowerCAmelCase_						=      tokenizer.num_special_tokens_to_add(_UpperCAmelCase)
            lowerCAmelCase_						=      compute_effective_axis_dimension(
                _UpperCAmelCase					,					fixed_dimension=OnnxConfig.default_fixed_sequence					,					num_token_to_add=_UpperCAmelCase)

            # Generate dummy inputs according to compute batch and sequence
            lowerCAmelCase_						=      [''' '''.join([tokenizer.unk_token]) * seq_length] * batch_size
            lowerCAmelCase_						=      dict(tokenizer(_UpperCAmelCase					,					return_tensors=_UpperCAmelCase))
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)

            elif self.task == "causal-lm":
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_causal_lm(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)
            else:
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)

            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      super()._flatten_past_key_values_(_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)
            else:
                  lowerCAmelCase_						=      super(_UpperCAmelCase					,					self)._flatten_past_key_values_(
                      _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)



