




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase_									=	logging.get_logger(__name__)

UpperCAmelCase_									=	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_									=	{
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_									=	{
    'google/realm-cc-news-pretrained-embedder': 512,
    'google/realm-cc-news-pretrained-encoder': 512,
    'google/realm-cc-news-pretrained-scorer': 512,
    'google/realm-cc-news-pretrained-openqa': 512,
    'google/realm-orqa-nq-openqa': 512,
    'google/realm-orqa-nq-reader': 512,
    'google/realm-orqa-wq-openqa': 512,
    'google/realm-orqa-wq-reader': 512,
}

UpperCAmelCase_									=	{
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}





class     lowerCamelCase__(					__lowerCamelCase):
      UpperCAmelCase__      :	List[str]    		=   VOCAB_FILES_NAMES
      UpperCAmelCase__      :	List[Any]    		=   PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__      :	Union[str, Any]    		=   PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase__      :	Any    		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__      :	List[Any]    		=   RealmTokenizer
      def __init__(				self: int    ,       UpperCamelCase_: Tuple=None    ,       UpperCamelCase_: List[str]=None    ,       UpperCamelCase_: List[str]=True    ,       UpperCamelCase_: Any="[UNK]"    ,       UpperCamelCase_: List[Any]="[SEP]"    ,       UpperCamelCase_: Tuple="[PAD]"    ,       UpperCamelCase_: Dict="[CLS]"    ,       UpperCamelCase_: Optional[int]="[MASK]"    ,       UpperCamelCase_: List[Any]=True    ,       UpperCamelCase_: str=None    ,       **UpperCamelCase_: Any    ,       ):
             super().__init__(
                 UpperCamelCase_    ,       tokenizer_file=UpperCamelCase_    ,       do_lower_case=UpperCamelCase_    ,       unk_token=UpperCamelCase_    ,       sep_token=UpperCamelCase_    ,       pad_token=UpperCamelCase_    ,       cls_token=UpperCamelCase_    ,       mask_token=UpperCamelCase_    ,       tokenize_chinese_chars=UpperCamelCase_    ,       strip_accents=UpperCamelCase_    ,       **UpperCamelCase_    ,       )

             __lowerCamelCase   =						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
             if (
                 normalizer_state.get("""lowercase"""    ,       UpperCamelCase_   ) != do_lower_case
                 or normalizer_state.get("""strip_accents"""    ,       UpperCamelCase_   ) != strip_accents
                 or normalizer_state.get("""handle_chinese_chars"""    ,       UpperCamelCase_   ) != tokenize_chinese_chars
             ):
                    __lowerCamelCase   =						getattr(UpperCamelCase_    ,       normalizer_state.pop("""type"""   )   )
                    __lowerCamelCase   =						do_lower_case
                    __lowerCamelCase   =						strip_accents
                    __lowerCamelCase   =						tokenize_chinese_chars
                    __lowerCamelCase   =						normalizer_class(**UpperCamelCase_   )

             __lowerCamelCase   =						do_lower_case
      def 			lowerCAmelCase__						(				self: Optional[Any]    ,       UpperCamelCase_: int    ,       **UpperCamelCase_: Union[str, Any]   ):
             __lowerCamelCase   =						PaddingStrategy.MAX_LENGTH

             __lowerCamelCase   =						text
             __lowerCamelCase   =						kwargs.pop("""text_pair"""    ,       UpperCamelCase_   )
             __lowerCamelCase   =						kwargs.pop("""return_tensors"""    ,       UpperCamelCase_   )

             __lowerCamelCase   =						{
                 """input_ids""": [],
                 """attention_mask""": [],
                 """token_type_ids""": [],
             }

             for idx, candidate_text in enumerate(UpperCamelCase_   ):
                    if batch_text_pair is not None:
                           __lowerCamelCase   =						batch_text_pair[idx]
                    else:
                           __lowerCamelCase   =						None

                    __lowerCamelCase   =						super().__call__(UpperCamelCase_    ,       UpperCamelCase_    ,       return_tensors=UpperCamelCase_    ,       **UpperCamelCase_   )

                    __lowerCamelCase   =						encoded_candidates.get("""input_ids"""   )
                    __lowerCamelCase   =						encoded_candidates.get("""attention_mask"""   )
                    __lowerCamelCase   =						encoded_candidates.get("""token_type_ids"""   )

                    if encoded_input_ids is not None:
                           output_data["input_ids"].append(UpperCamelCase_   )
                    if encoded_attention_mask is not None:
                           output_data["attention_mask"].append(UpperCamelCase_   )
                    if encoded_token_type_ids is not None:
                           output_data["token_type_ids"].append(UpperCamelCase_   )

             __lowerCamelCase   =						{key: item for key, item in output_data.items() if len(UpperCamelCase_   ) != 0}

             return BatchEncoding(UpperCamelCase_    ,       tensor_type=UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Dict    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: Optional[Any]=None   ):
             __lowerCamelCase   =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output
      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: List[int]    ,       UpperCamelCase_: Optional[List[int]] = None   ):
             __lowerCamelCase   =						[self.sep_token_id]
             __lowerCamelCase   =						[self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep   ) * [0]
             return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Optional[str] = None   ):
             __lowerCamelCase   =						self._tokenizer.model.save(UpperCamelCase_    ,       name=UpperCamelCase_   )
             return tuple(UpperCamelCase_   )







from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch





@require_torch
@require_tf
class     lowerCamelCase__(					__lowerCamelCase):
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             __lowerCamelCase   =						SMALL_MODEL_IDENTIFIER
             __lowerCamelCase   =						"""pt"""
             __lowerCamelCase   =						"""tf"""
      def 			lowerCAmelCase__						(				self: Optional[Any]    ,       UpperCamelCase_: Optional[Any]   ):
             __lowerCamelCase   =						AutoModel.from_pretrained(self.test_model   )
             model_pt.save_pretrained(UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Tuple    ,       UpperCamelCase_: Union[str, Any]   ):
             __lowerCamelCase   =						TFAutoModel.from_pretrained(self.test_model    ,       from_pt=UpperCamelCase_   )
             model_tf.save_pretrained(UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Union[str, Any]   ):
             __lowerCamelCase   =						"""mock_framework"""

             # Framework provided - return whatever the user provides
             __lowerCamelCase   =						FeaturesManager.determine_framework(self.test_model    ,       UpperCamelCase_   )
             self.assertEqual(UpperCamelCase_    ,       UpperCamelCase_   )

             # Local checkpoint and framework provided - return provided framework
             # PyTorch checkpoint
             with TemporaryDirectory() as local_pt_ckpt:
                    self._setup_pt_ckpt(UpperCamelCase_   )
                    __lowerCamelCase   =						FeaturesManager.determine_framework(UpperCamelCase_    ,       UpperCamelCase_   )
                    self.assertEqual(UpperCamelCase_    ,       UpperCamelCase_   )

             # TensorFlow checkpoint
             with TemporaryDirectory() as local_tf_ckpt:
                    self._setup_tf_ckpt(UpperCamelCase_   )
                    __lowerCamelCase   =						FeaturesManager.determine_framework(UpperCamelCase_    ,       UpperCamelCase_   )
                    self.assertEqual(UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: int   ):
             # PyTorch checkpoint
             with TemporaryDirectory() as local_pt_ckpt:
                    self._setup_pt_ckpt(UpperCamelCase_   )
                    __lowerCamelCase   =						FeaturesManager.determine_framework(UpperCamelCase_   )
                    self.assertEqual(UpperCamelCase_    ,       self.framework_pt   )

             # TensorFlow checkpoint
             with TemporaryDirectory() as local_tf_ckpt:
                    self._setup_tf_ckpt(UpperCamelCase_   )
                    __lowerCamelCase   =						FeaturesManager.determine_framework(UpperCamelCase_   )
                    self.assertEqual(UpperCamelCase_    ,       self.framework_tf   )

             # Invalid local checkpoint
             with TemporaryDirectory() as local_invalid_ckpt:
                    with self.assertRaises(UpperCamelCase_   ):
                           __lowerCamelCase   =						FeaturesManager.determine_framework(UpperCamelCase_   )




      def 			lowerCAmelCase__						(				self: Optional[int]   ):
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             with patch("""transformers.onnx.features.is_tf_available"""    ,       UpperCamelCase_   ):
                    __lowerCamelCase   =						FeaturesManager.determine_framework(self.test_model   )
                    self.assertEqual(UpperCamelCase_    ,       self.framework_pt   )

             # PyTorch not in environment -> use TensorFlow
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             with patch("""transformers.onnx.features.is_torch_available"""    ,       UpperCamelCase_   ):
                    __lowerCamelCase   =						FeaturesManager.determine_framework(self.test_model   )
                    self.assertEqual(UpperCamelCase_    ,       self.framework_tf   )

             # Both in environment -> use PyTorch
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             with patch("""transformers.onnx.features.is_tf_available"""    ,       UpperCamelCase_   ), patch(
                 """transformers.onnx.features.is_torch_available"""    ,       UpperCamelCase_   ):
                    __lowerCamelCase   =						FeaturesManager.determine_framework(self.test_model   )
                    self.assertEqual(UpperCamelCase_    ,       self.framework_pt   )

             # Both not in environment -> raise error
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             __lowerCamelCase   =						MagicMock(return_value=UpperCamelCase_   )
             with patch("""transformers.onnx.features.is_tf_available"""    ,       UpperCamelCase_   ), patch(
                 """transformers.onnx.features.is_torch_available"""    ,       UpperCamelCase_   ):
                    with self.assertRaises(UpperCamelCase_   ):
                           __lowerCamelCase   =						FeaturesManager.determine_framework(self.test_model   )

