


"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase_			     =  logging.get_logger(__name__)

lowercase_			     =  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase_			     =  {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

lowercase_			     =  {
    "google/realm-cc-news-pretrained-embedder": 5_1_2,
    "google/realm-cc-news-pretrained-encoder": 5_1_2,
    "google/realm-cc-news-pretrained-scorer": 5_1_2,
    "google/realm-cc-news-pretrained-openqa": 5_1_2,
    "google/realm-orqa-nq-openqa": 5_1_2,
    "google/realm-orqa-nq-reader": 5_1_2,
    "google/realm-orqa-wq-openqa": 5_1_2,
    "google/realm-orqa-wq-reader": 5_1_2,
}

lowercase_			     =  {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}
class 					__lowerCAmelCase					(   __SCREAMING_SNAKE_CASE       ):


		'''simple docstring'''



		__UpperCAmelCase		:			List[str]										=				VOCAB_FILES_NAMES
		__UpperCAmelCase		:			Optional[int]										=				PRETRAINED_VOCAB_FILES_MAP
		__UpperCAmelCase		:			Optional[int]										=				PRETRAINED_INIT_CONFIGURATION
		__UpperCAmelCase		:			int										=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__UpperCAmelCase		:			Tuple										=				RealmTokenizer

		def __init__(  self							, _a=None							, _a=None							, _a=True							, _a="[UNK]"							, _a="[SEP]"							, _a="[PAD]"							, _a="[CLS]"							, _a="[MASK]"							, _a=True							, _a=None							, **_a							, ):
				super().__init__(
				    _a							, tokenizer_file=_a							, do_lower_case=_a							, unk_token=_a							, sep_token=_a							, pad_token=_a							, cls_token=_a							, mask_token=_a							, tokenize_chinese_chars=_a							, strip_accents=_a							, **_a							, )

				__a           =			json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
				if (
				    normalizer_state.get('''lowercase'''							, _a      ) != do_lower_case
				    or normalizer_state.get('''strip_accents'''							, _a      ) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''							, _a      ) != tokenize_chinese_chars
				):
						__a           =			getattr(_a							, normalizer_state.pop('''type'''      )      )
						__a           =			do_lower_case
						__a           =			strip_accents
						__a           =			tokenize_chinese_chars
						__a           =			normalizer_class(**_a      )

				__a           =			do_lower_case

		def 			__UpperCAmelCase       (  self							, _a							, **_a      ):
				__a           =			PaddingStrategy.MAX_LENGTH

				__a           =			text
				__a           =			kwargs.pop('''text_pair'''							, _a      )
				__a           =			kwargs.pop('''return_tensors'''							, _a      )

				__a           =			{
				    '''input_ids''': [],
				    '''attention_mask''': [],
				    '''token_type_ids''': [],
				}

				for idx, candidate_text in enumerate(_a      ):
						if batch_text_pair is not None:
								__a           =			batch_text_pair[idx]
						else:
								__a           =			None

						__a           =			super().__call__(_a							, _a							, return_tensors=_a							, **_a      )

						__a           =			encoded_candidates.get('''input_ids'''      )
						__a           =			encoded_candidates.get('''attention_mask'''      )
						__a           =			encoded_candidates.get('''token_type_ids'''      )

						if encoded_input_ids is not None:
								output_data["input_ids"].append(_a      )
						if encoded_attention_mask is not None:
								output_data["attention_mask"].append(_a      )
						if encoded_token_type_ids is not None:
								output_data["token_type_ids"].append(_a      )

				__a           =			{key: item for key, item in output_data.items() if len(_a      ) != 0}

				return BatchEncoding(_a							, tensor_type=_a      )

		def 			__UpperCAmelCase       (  self							, _a							, _a=None      ):
				__a           =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

				return output

		def 			__UpperCAmelCase       (  self							, _a							, _a = None      ):
				__a           =			[self.sep_token_id]
				__a           =			[self.cls_token_id]
				if token_ids_a is None:
						return len(cls + token_ids_a + sep      ) * [0]
				return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





		def 			__UpperCAmelCase       (  self							, _a							, _a = None      ):
				__a           =			self._tokenizer.model.save(_a							, name=_a      )
				return tuple(_a      )







'''simple docstring'''




import math
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import SchedulerMixin, SchedulerOutput

class 	lowerCamelCase_	(snake_case__							, snake_case__    ):
      '''simple docstring'''

      __UpperCamelCase:  Optional[Any]						=						1




      @register_to_config
      def __init__(       self						:				Optional[int]						,					A						:				int = 1000						,					A						:				Optional[Union[np.ndarray, List[float]]] = None       ):
             # set `betas`, `alphas`, `timesteps`
             self.set_timesteps(A       )

             # standard deviation of the initial noise distribution
             _UpperCAmelCase     :  int							      =    1.0

             # For now we only support F-PNDM, i.e. the runge-kutta method
             # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf
             # mainly at formula (9), (12), (13) and the Algorithm 2.
             _UpperCAmelCase     :  int							      =    4

             # running values
             _UpperCAmelCase     :  Dict							      =    []




      def      _A	(       self						:				Optional[int]						,					A						:				int						,					A						:				Union[str, torch.device] = None       ):
             _UpperCAmelCase     :  int							      =    num_inference_steps
             _UpperCAmelCase     :  Union[str, Any]							      =    torch.linspace(1						,					0						,					num_inference_steps + 1       )[:-1]
             _UpperCAmelCase     :  Any							      =    torch.cat([steps, torch.tensor([0.0]       )]       )

             if self.config.trained_betas is not None:
                    _UpperCAmelCase     :  str							      =    torch.tensor(self.config.trained_betas						,					dtype=torch.floataa       )
             else:
                    _UpperCAmelCase     :  Dict							      =    torch.sin(steps * math.pi / 2       ) ** 2

             _UpperCAmelCase     :  List[Any]							      =    (1.0 - self.betas**2) ** 0.5

             _UpperCAmelCase     :  List[str]							      =    (torch.atana(self.betas						,					self.alphas       ) / math.pi * 2)[:-1]
             _UpperCAmelCase     :  Dict							      =    timesteps.to(A       )

             _UpperCAmelCase     :  Dict							      =    []




      def      _A	(       self						:				Optional[int]						,					A						:				torch.FloatTensor						,					A						:				int						,					A						:				torch.FloatTensor						,					A						:				bool = True						,					):
             if self.num_inference_steps is None:
                    raise ValueError(
                        "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"       )

             _UpperCAmelCase     :  Tuple							      =    (self.timesteps == timestep).nonzero().item()
             _UpperCAmelCase     :  Optional[Any]							      =    timestep_index + 1

             _UpperCAmelCase     :  int							      =    sample * self.betas[timestep_index] + model_output * self.alphas[timestep_index]
             self.ets.append(A       )

             if len(self.ets       ) == 1:
                    _UpperCAmelCase     :  List[Any]							      =    self.ets[-1]
             elif len(self.ets       ) == 2:
                    _UpperCAmelCase     :  str							      =    (3 * self.ets[-1] - self.ets[-2]) / 2
             elif len(self.ets       ) == 3:
                    _UpperCAmelCase     :  Tuple							      =    (23 * self.ets[-1] - 16 * self.ets[-2] + 5 * self.ets[-3]) / 12
             else:
                    _UpperCAmelCase     :  Union[str, Any]							      =    (1 / 24) * (55 * self.ets[-1] - 59 * self.ets[-2] + 37 * self.ets[-3] - 9 * self.ets[-4])

             _UpperCAmelCase     :  Union[str, Any]							      =    self._get_prev_sample(A						,					A						,					A						,					A       )

             if not return_dict:
                    return (prev_sample,)

             return SchedulerOutput(prev_sample=A       )




      def      _A	(       self						:				Union[str, Any]						,					A						:				torch.FloatTensor						,					*A						:				Union[str, Any]						,					**A						:				Dict       ):
             return sample




      def      _A	(       self						:				Optional[Any]						,					A						:				Optional[int]						,					A						:				int						,					A						:				Optional[Any]						,					A						:				List[str]       ):
             _UpperCAmelCase     :  List[str]							      =    self.alphas[timestep_index]
             _UpperCAmelCase     :  List[Any]							      =    self.betas[timestep_index]

             _UpperCAmelCase     :  Optional[Any]							      =    self.alphas[prev_timestep_index]
             _UpperCAmelCase     :  Dict							      =    self.betas[prev_timestep_index]

             _UpperCAmelCase     :  Tuple							      =    (sample - sigma * ets) / max(A						,					1E-8       )
             _UpperCAmelCase     :  List[str]							      =    next_alpha * pred + ets * next_sigma

             return prev_sample





      def __len__(       self						:				Union[str, Any]       ):
             return self.config.num_train_timesteps

