





import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger



class 					A__   (       __snake_case			):





      def __init__( self       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					List[str]       	=   params

       UpperCamelCase	:					Union[str, Any]       	=   np.array(A_   )
       UpperCamelCase	:					Optional[int]       	=   np.array([len(A_   ) for t in data]   )

       self.check()
       self.remove_long_sequences()
       self.remove_empty_sequences()
       self.remove_unknown_sequences()
       self.check()
       self.print_statistics()





      def __getitem__( self       ,							A_   ):
       '''simple docstring'''




       return (self.token_ids[index], self.lengths[index])





      def __len__( self   ):
       '''simple docstring'''




       return len(self.lengths   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       assert len(self.token_ids   ) == len(self.lengths   )
       assert all(self.lengths[i] == len(self.token_ids[i]   ) for i in range(len(self.lengths   )   )   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   self.params.max_model_input_size
       UpperCamelCase	:					Dict       	=   self.lengths > max_len
       logger.info(F"""Splitting {sum(A_   )} too long sequences."""   )

       def divide_chunks(A_       ,							A_   ):
        return [l[i : i + n] for i in range(0       ,							len(A_   )       ,							A_   )]

       UpperCamelCase	:					List[str]       	=   []
       UpperCamelCase	:					Tuple       	=   []
       if self.params.mlm:
        UpperCamelCase	,    UpperCamelCase	:					Dict       	=   self.params.special_tok_ids["cls_token"], self.params.special_tok_ids["sep_token"]
       else:
        UpperCamelCase	,    UpperCamelCase	:					Dict       	=   self.params.special_tok_ids["bos_token"], self.params.special_tok_ids["eos_token"]

       for seq_, len_ in zip(self.token_ids       ,							self.lengths   ):
        assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
        if len_ <= max_len:
         new_tok_ids.append(seq_   )
         new_lengths.append(len_   )
        else:
         UpperCamelCase	:					Dict       	=   []
         for sub_s in divide_chunks(seq_       ,							max_len - 2   ):
          if sub_s[0] != cls_id:
           UpperCamelCase	:					Union[str, Any]       	=   np.insert(A_       ,							0       ,							A_   )
          if sub_s[-1] != sep_id:
           UpperCamelCase	:					Optional[int]       	=   np.insert(A_       ,							len(A_   )       ,							A_   )
          assert len(A_   ) <= max_len
          assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
          sub_seqs.append(A_   )

         new_tok_ids.extend(A_   )
         new_lengths.extend([len(A_   ) for l in sub_seqs]   )

       UpperCamelCase	:					Union[str, Any]       	=   np.array(A_   )
       UpperCamelCase	:					Union[str, Any]       	=   np.array(A_   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Tuple       	=   len(self   )
       UpperCamelCase	:					Dict       	=   self.lengths > 11
       UpperCamelCase	:					List[str]       	=   self.token_ids[indices]
       UpperCamelCase	:					Tuple       	=   self.lengths[indices]
       UpperCamelCase	:					Optional[Any]       	=   len(self   )
       logger.info(F"""Remove {init_size - new_size} too short (<=11 tokens) sequences."""   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       if "unk_token" not in self.params.special_tok_ids:
        return
       else:
        UpperCamelCase	:					List[str]       	=   self.params.special_tok_ids["unk_token"]
       UpperCamelCase	:					int       	=   len(self   )
       UpperCamelCase	:					str       	=   np.array([np.count_nonzero(a == unk_token_id   ) for a in self.token_ids]   )
       UpperCamelCase	:					List[Any]       	=   (unk_occs / self.lengths) < 0.5
       UpperCamelCase	:					List[Any]       	=   self.token_ids[indices]
       UpperCamelCase	:					str       	=   self.lengths[indices]
       UpperCamelCase	:					Dict       	=   len(self   )
       logger.info(F"""Remove {init_size - new_size} sequences with a high level of unknown tokens (50%)."""   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       if not self.params.is_master:
        return
       logger.info(F"""{len(self   )} sequences"""   )
       # data_len = sum(self.lengths)
       # nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
       # logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

       # unk_idx = self.params.special_tok_ids['unk_token']
       # nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
       # logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					str       	=   [t[0] for t in batch]
       UpperCamelCase	:					str       	=   [t[1] for t in batch]
       assert len(A_   ) == len(A_   )

       # Max for paddings
       UpperCamelCase	:					Union[str, Any]       	=   max(A_   )

       # Pad token ids
       if self.params.mlm:
        UpperCamelCase	:					List[Any]       	=   self.params.special_tok_ids["pad_token"]
       else:
        UpperCamelCase	:					Dict       	=   self.params.special_tok_ids["unk_token"]
       UpperCamelCase	:					Optional[int]       	=   [list(t.astype(A_   )   ) + [pad_idx] * (max_seq_len_ - len(A_   )) for t in token_ids]
       assert len(tk_   ) == len(A_   )
       assert all(len(A_   ) == max_seq_len_ for t in tk_   )

       UpperCamelCase	:					Any       	=   torch.tensor(tk_   )  # (bs, max_seq_len_)
       UpperCamelCase	:					Any       	=   torch.tensor(A_   )  # (bs)
       return tk_t, lg_t







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowerCamelCase       :     Any   =     {
    """configuration_electra""": ["""ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ElectraConfig""", """ElectraOnnxConfig"""],
    """tokenization_electra""": ["""ElectraTokenizer"""],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase       :     Dict   =     ["""ElectraTokenizerFast"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase       :     Tuple   =     [
        """ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """ElectraForCausalLM""",
        """ElectraForMaskedLM""",
        """ElectraForMultipleChoice""",
        """ElectraForPreTraining""",
        """ElectraForQuestionAnswering""",
        """ElectraForSequenceClassification""",
        """ElectraForTokenClassification""",
        """ElectraModel""",
        """ElectraPreTrainedModel""",
        """load_tf_weights_in_electra""",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase       :     List[Any]   =     [
        """TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """TFElectraForMaskedLM""",
        """TFElectraForMultipleChoice""",
        """TFElectraForPreTraining""",
        """TFElectraForQuestionAnswering""",
        """TFElectraForSequenceClassification""",
        """TFElectraForTokenClassification""",
        """TFElectraModel""",
        """TFElectraPreTrainedModel""",
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCamelCase       :     List[str]   =     [
        """FlaxElectraForCausalLM""",
        """FlaxElectraForMaskedLM""",
        """FlaxElectraForMultipleChoice""",
        """FlaxElectraForPreTraining""",
        """FlaxElectraForQuestionAnswering""",
        """FlaxElectraForSequenceClassification""",
        """FlaxElectraForTokenClassification""",
        """FlaxElectraModel""",
        """FlaxElectraPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
    from .tokenization_electra import ElectraTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_electra_fast import ElectraTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_electra import (
            ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            ElectraForCausalLM,
            ElectraForMaskedLM,
            ElectraForMultipleChoice,
            ElectraForPreTraining,
            ElectraForQuestionAnswering,
            ElectraForSequenceClassification,
            ElectraForTokenClassification,
            ElectraModel,
            ElectraPreTrainedModel,
            load_tf_weights_in_electra,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_electra import (
            TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFElectraForMaskedLM,
            TFElectraForMultipleChoice,
            TFElectraForPreTraining,
            TFElectraForQuestionAnswering,
            TFElectraForSequenceClassification,
            TFElectraForTokenClassification,
            TFElectraModel,
            TFElectraPreTrainedModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_electra import (
            FlaxElectraForCausalLM,
            FlaxElectraForMaskedLM,
            FlaxElectraForMultipleChoice,
            FlaxElectraForPreTraining,
            FlaxElectraForQuestionAnswering,
            FlaxElectraForSequenceClassification,
            FlaxElectraForTokenClassification,
            FlaxElectraModel,
            FlaxElectraPreTrainedModel,
        )

else:
    import sys

    __lowerCamelCase       :     Any   =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
