

'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase       =     logging.get_logger(__name__)

UpperCAmelCase       =     {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase       =     {
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase       =     {
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

UpperCAmelCase       =     {
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}


class      lowerCAmelCase   (					_A      ):
       lowerCAmelCase_				=       VOCAB_FILES_NAMES
       lowerCAmelCase_				=       PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase_				=       PRETRAINED_INIT_CONFIGURATION
       lowerCAmelCase_				=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase_				=       RealmTokenizer







       def __init__(							self	:       str  ,					__lowercase	:       Optional[int]=None  ,					__lowercase	:       int=None  ,					__lowercase	:       Union[str, Any]=True  ,					__lowercase	:       str="[UNK]"  ,					__lowercase	:       Dict="[SEP]"  ,					__lowercase	:       Dict="[PAD]"  ,					__lowercase	:       List[str]="[CLS]"  ,					__lowercase	:       List[str]="[MASK]"  ,					__lowercase	:       Dict=True  ,					__lowercase	:       Union[str, Any]=None  ,					**__lowercase	:       int  ,					):







            """simple docstring"""







            super().__init__(
                __lowercase  ,					tokenizer_file=__lowercase  ,					do_lower_case=__lowercase  ,					unk_token=__lowercase  ,					sep_token=__lowercase  ,					pad_token=__lowercase  ,					cls_token=__lowercase  ,					mask_token=__lowercase  ,					tokenize_chinese_chars=__lowercase  ,					strip_accents=__lowercase  ,					**__lowercase  ,					)

            __lowercase							=json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
            if (
                normalizer_state.get('lowercase'  ,					__lowercase					) != do_lower_case
                or normalizer_state.get('strip_accents'  ,					__lowercase					) != strip_accents
                or normalizer_state.get('handle_chinese_chars'  ,					__lowercase					) != tokenize_chinese_chars
            ):
                 __lowercase							=getattr(__lowercase  ,					normalizer_state.pop('type'					)					)
                 __lowercase							=do_lower_case
                 __lowercase							=strip_accents
                 __lowercase							=tokenize_chinese_chars
                 __lowercase							=normalizer_class(**__lowercase					)

            __lowercase							=do_lower_case







       def 	snake_case      (							self	:       Union[str, Any]  ,					__lowercase	:       int  ,					**__lowercase	:       Dict					):







            """simple docstring"""







            __lowercase							=PaddingStrategy.MAX_LENGTH

            __lowercase							=text
            __lowercase							=kwargs.pop('text_pair'  ,					__lowercase					)
            __lowercase							=kwargs.pop('return_tensors'  ,					__lowercase					)

            __lowercase							={
                '''input_ids''': [],
                '''attention_mask''': [],
                '''token_type_ids''': [],
            }

            for idx, candidate_text in enumerate(__lowercase					):
                 if batch_text_pair is not None:
                      __lowercase							=batch_text_pair[idx]
                 else:
                      __lowercase							=None

                 __lowercase							=super().__call__(__lowercase  ,					__lowercase  ,					return_tensors=__lowercase  ,					**__lowercase					)

                 __lowercase							=encoded_candidates.get('input_ids'					)
                 __lowercase							=encoded_candidates.get('attention_mask'					)
                 __lowercase							=encoded_candidates.get('token_type_ids'					)

                 if encoded_input_ids is not None:
                      output_data["input_ids"].append(__lowercase					)
                 if encoded_attention_mask is not None:
                      output_data["attention_mask"].append(__lowercase					)
                 if encoded_token_type_ids is not None:
                      output_data["token_type_ids"].append(__lowercase					)

            __lowercase							={key: item for key, item in output_data.items() if len(__lowercase					) != 0}

            return BatchEncoding(__lowercase  ,					tensor_type=__lowercase					)







       def 	snake_case      (							self	:       Any  ,					__lowercase	:       Union[str, Any]  ,					__lowercase	:       Optional[int]=None					):







            """simple docstring"""







            __lowercase							=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

            return output







       def 	snake_case      (							self	:       List[str]  ,					__lowercase	:       List[int]  ,					__lowercase	:       Optional[List[int]] = None					):







            """simple docstring"""







            __lowercase							=[self.sep_token_id]
            __lowercase							=[self.cls_token_id]
            if token_ids_a is None:
                 return len(cls + token_ids_a + sep					) * [0]
            return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







       def 	snake_case      (							self	:       List[Any]  ,					__lowercase	:       str  ,					__lowercase	:       Optional[str] = None					):







            """simple docstring"""







            __lowercase							=self._tokenizer.model.save(__lowercase  ,					name=__lowercase					)
            return tuple(__lowercase					)










"""simple docstring"""



# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
						import torch





class 					_UpperCAmelCase       ( TensorFormatter[Mapping, "torch.Tensor", Mapping]							):




					def __init__(				self     :						Any    ,						A     :						int=None    ,						**A     :						str   )  ->  Union[str, Any]:
							super().__init__(features=A   )
							lowercase_ : Union[str, Any]							       =   torch_tensor_kwargs
							import torch  # noqa import torch at initialization




					def   A  (				self     :						Dict    ,						A     :						int   )  ->  List[Any]:
							import torch

							if isinstance(A    ,						A   ) and column:
									if all(
									    isinstance(A    ,						torch.Tensor   ) and x.shape == column[0].shape and x.dtype == column[0].dtype
									    for x in column   ):
											return torch.stack(A   )
							return column




					def   A  (				self     :						int    ,						A     :						Any   )  ->  Optional[Any]:
							import torch

							if isinstance(A    ,						(str, bytes, type(A   ))   ):
									return value
							elif isinstance(A    ,						(np.character, np.ndarray)   ) and np.issubdtype(value.dtype    ,						np.character   ):
									return value.tolist()

							lowercase_ : Any							       =   {}

							if isinstance(A    ,						(np.number, np.ndarray)   ) and np.issubdtype(value.dtype    ,						np.integer   ):
									lowercase_ : Any							       =   {'''dtype''': torch.intaa}
							elif isinstance(A    ,						(np.number, np.ndarray)   ) and np.issubdtype(value.dtype    ,						np.floating   ):
									lowercase_ : Dict							       =   {'''dtype''': torch.floataa}
							elif config.PIL_AVAILABLE and "PIL" in sys.modules:
									import PIL.Image

									if isinstance(A    ,						PIL.Image.Image   ):
											lowercase_ : Dict							       =   np.asarray(A   )
							return torch.tensor(A    ,						**{**default_dtype, **self.torch_tensor_kwargs}   )




					def   A  (				self     :						Union[str, Any]    ,						A     :						Optional[int]   )  ->  str:
							import torch

							# support for torch, tf, jax etc.
							if hasattr(A    ,						'''__array__'''   ) and not isinstance(A    ,						torch.Tensor   ):
									lowercase_ : Optional[int]							       =   data_struct.__array__()
							# support for nested types like struct of list of struct
							if isinstance(A    ,						np.ndarray   ):
									if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
											return self._consolidate([self.recursive_tensorize(A   ) for substruct in data_struct]   )
							elif isinstance(A    ,						(list, tuple)   ):
									return self._consolidate([self.recursive_tensorize(A   ) for substruct in data_struct]   )
							return self._tensorize(A   )




					def   A  (				self     :						Dict    ,						A     :						dict   )  ->  Tuple:
							return map_nested(self._recursive_tensorize    ,						A    ,						map_list=A   )




					def   A  (				self     :						str    ,						A     :						pa.Table   )  ->  Mapping:
							lowercase_ : Optional[Any]							       =   self.numpy_arrow_extractor().extract_row(A   )
							lowercase_ : str							       =   self.python_features_decoder.decode_row(A   )
							return self.recursive_tensorize(A   )




					def   A  (				self     :						List[Any]    ,						A     :						pa.Table   )  ->  "torch.Tensor":
							lowercase_ : List[str]							       =   self.numpy_arrow_extractor().extract_column(A   )
							lowercase_ : str							       =   self.python_features_decoder.decode_column(A    ,						pa_table.column_names[0]   )
							lowercase_ : Optional[int]							       =   self.recursive_tensorize(A   )
							lowercase_ : Any							       =   self._consolidate(A   )
							return column






					def   A  (				self     :						List[str]    ,						A     :						pa.Table   )  ->  Mapping:
							lowercase_ : Optional[int]							       =   self.numpy_arrow_extractor().extract_batch(A   )
							lowercase_ : int							       =   self.python_features_decoder.decode_batch(A   )
							lowercase_ : Dict							       =   self.recursive_tensorize(A   )
							for column_name in batch:
									lowercase_ : Optional[Any]							       =   self._consolidate(batch[column_name]   )
							return batch



