

import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter







class 					__lowerCamelCase							(      TensorFormatter[Mapping, np.ndarray, Mapping]      ):







      def __init__(	self				,    lowerCamelCase=None				,    **lowerCamelCase  )		->			str:
        super().__init__(features=_lowerCAmelCase  )
        snake_case_					= np_array_kwargs







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			List[str]:
        if isinstance(_lowerCAmelCase				,    _lowerCAmelCase  ):
          if column and all(
              isinstance(_lowerCAmelCase				,    np.ndarray  ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column  ):
            return np.stack(_lowerCAmelCase  )
          else:
            # don't use np.array(column, dtype=object)
            # since it fails in certain cases
            # see https://stackoverflow.com/q/51005699
            snake_case_					= np.empty(len(_lowerCAmelCase  )				,    dtype=_lowerCAmelCase  )
            snake_case_					= column
            return out
        return column







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			List[Any]:
        if isinstance(_lowerCAmelCase				,    (str, bytes, type(_lowerCAmelCase  ))  ):
          return value
        elif isinstance(_lowerCAmelCase				,    (np.character, np.ndarray)  ) and np.issubdtype(value.dtype				,    np.character  ):
          return value
        elif isinstance(_lowerCAmelCase				,    np.number  ):
          return value

        snake_case_					= {}

        if isinstance(_lowerCAmelCase				,    np.ndarray  ) and np.issubdtype(value.dtype				,    np.integer  ):
          snake_case_					= {"""dtype""": np.intaa}
        elif isinstance(_lowerCAmelCase				,    np.ndarray  ) and np.issubdtype(value.dtype				,    np.floating  ):
          snake_case_					= {"""dtype""": np.floataa}
        elif config.PIL_AVAILABLE and "PIL" in sys.modules:
          import PIL.Image

          if isinstance(_lowerCAmelCase				,    PIL.Image.Image  ):
            return np.asarray(_lowerCAmelCase				,    **self.np_array_kwargs  )

        return np.asarray(_lowerCAmelCase				,    **{**default_dtype, **self.np_array_kwargs}  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			int:
        # support for torch, tf, jax etc.
        if config.TORCH_AVAILABLE and "torch" in sys.modules:
          import torch

          if isinstance(_lowerCAmelCase				,    torch.Tensor  ):
            return self._tensorize(data_struct.detach().cpu().numpy()[()]  )
        if hasattr(_lowerCAmelCase				,    """__array__"""  ) and not isinstance(_lowerCAmelCase				,    (np.ndarray, np.character, np.number)  ):
          snake_case_					= data_struct.__array__()
        # support for nested types like struct of list of struct
        if isinstance(_lowerCAmelCase				,    np.ndarray  ):
          if data_struct.dtype == object:
            return self._consolidate([self.recursive_tensorize(_lowerCAmelCase  ) for substruct in data_struct]  )
        if isinstance(_lowerCAmelCase				,    (list, tuple)  ):
          return self._consolidate([self.recursive_tensorize(_lowerCAmelCase  ) for substruct in data_struct]  )
        return self._tensorize(_lowerCAmelCase  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			Optional[Any]:
        return map_nested(self._recursive_tensorize				,    _lowerCAmelCase				,    map_list=_lowerCAmelCase  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			int:
        snake_case_					= self.numpy_arrow_extractor().extract_row(_lowerCAmelCase  )
        snake_case_					= self.python_features_decoder.decode_row(_lowerCAmelCase  )
        return self.recursive_tensorize(_lowerCAmelCase  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			Any:
        snake_case_					= self.numpy_arrow_extractor().extract_column(_lowerCAmelCase  )
        snake_case_					= self.python_features_decoder.decode_column(_lowerCAmelCase				,    pa_table.column_names[0]  )
        snake_case_					= self.recursive_tensorize(_lowerCAmelCase  )
        snake_case_					= self._consolidate(_lowerCAmelCase  )
        return column







      def       lowerCAmelCase_       (	self				,    lowerCamelCase  )		->			Union[str, Any]:
        snake_case_					= self.numpy_arrow_extractor().extract_batch(_lowerCAmelCase  )
        snake_case_					= self.python_features_decoder.decode_batch(_lowerCAmelCase  )
        snake_case_					= self.recursive_tensorize(_lowerCAmelCase  )
        for column_name in batch:
          snake_case_					= self._consolidate(batch[column_name]  )
        return batch






# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus

# 1. pip install sentencepiece
#
# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt

# 3. build
import sentencepiece as spm

# pegasus:
# 1. no bos
# 2. eos_id is 1
# 3. unk_id is 2
# build a sample spm file accordingly
spm.SentencePieceTrainer.train(
    "--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000"
)

# 4. now update the fixture
# mv test_sentencepiece_no_bos.model ../../tests/fixtures/

