



'''simple docstring'''


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase	:       Tuple      	=	logging.get_logger(__name__)


_lowerCamelCase	:       Any      	=	{
    "nielsr/canine-s": 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_lowerCamelCase	:       int      	=	111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_lowerCamelCase	:       Any      	=	0

_lowerCamelCase	:       List[Any]      	=	0xE000
_lowerCamelCase	:       str      	=	0xE001
_lowerCamelCase	:       str      	=	0xE002
_lowerCamelCase	:       Dict      	=	0xE003
_lowerCamelCase	:       str      	=	0xE004

# Maps special codepoints to human-readable names.
_lowerCamelCase	:       Dict[int, str]      	=	{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_lowerCamelCase	:       Dict[str, int]      	=	{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    _SCREAMING_SNAKE_CASE	  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






    def __init__(  self      :					List[str]  ,						UpperCamelCase__      :					Tuple=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					Dict=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					Optional[int]=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					int=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					Optional[int]=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					Tuple=chr(UpperCamelCase__ )  ,						UpperCamelCase__      :					Tuple=False  ,						UpperCamelCase__      :					List[str]=2_0_4_8  ,						**UpperCamelCase__      :					Any  ,						):


       """simple docstring"""







       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else bos_token
       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else eos_token
       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else sep_token
       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else cls_token
       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       UpperCamelCase					=  AddedToken(UpperCamelCase__  ,						lstrip=UpperCamelCase__  ,						rstrip=UpperCamelCase__ ) if isinstance(UpperCamelCase__  ,						UpperCamelCase__ ) else mask_token

       super().__init__(
           bos_token=UpperCamelCase__  ,						eos_token=UpperCamelCase__  ,						sep_token=UpperCamelCase__  ,						cls_token=UpperCamelCase__  ,						pad_token=UpperCamelCase__  ,						mask_token=UpperCamelCase__  ,						add_prefix_space=UpperCamelCase__  ,						model_max_length=UpperCamelCase__  ,						**UpperCamelCase__  ,						)

       # Creates a mapping for looking up the IDs of special symbols.
       UpperCamelCase					=  {}
       for codepoint, name in SPECIAL_CODEPOINTS.items():
          UpperCamelCase					=  codepoint

       # Creates a mapping for looking up the string forms of special symbol IDs.
       UpperCamelCase					=  {
           codepoint: name for name, codepoint in self._special_codepoints.items()
       }

       UpperCamelCase					=  UNICODE_VOCAB_SIZE
       UpperCamelCase					=  len(self._special_codepoints )






    @property
    def  A       (  self      :					int ):


       """simple docstring"""







       return self._unicode_vocab_size






    def  A       (  self      :					List[Any]  ,						UpperCamelCase__      :					str ):


       """simple docstring"""







       return list(UpperCamelCase__ )






    def  A       (  self      :					Optional[int]  ,						UpperCamelCase__      :					str ):


       """simple docstring"""







       try:
          return ord(UpperCamelCase__ )
       except TypeError:
          raise ValueError(f"""invalid token: '{token}'""" )






    def  A       (  self      :					str  ,						UpperCamelCase__      :					int ):


       """simple docstring"""







       try:
          if index in SPECIAL_CODEPOINTS:
             return SPECIAL_CODEPOINTS[index]
          return chr(UpperCamelCase__ )
       except TypeError:
          raise ValueError(f"""invalid id: {index}""" )






    def  A       (  self      :					Union[str, Any]  ,						UpperCamelCase__      :					Optional[Any] ):


       """simple docstring"""







       return "".join(UpperCamelCase__ )






    def  A       (  self      :					Optional[int]  ,						UpperCamelCase__      :					List[int]  ,						UpperCamelCase__      :					Optional[List[int]] = None ):


       """simple docstring"""







       UpperCamelCase					=  [self.sep_token_id]
       UpperCamelCase					=  [self.cls_token_id]

       UpperCamelCase					=  cls + token_ids_a + sep
       if token_ids_a is not None:
          result += token_ids_a + sep
       return result






    def  A       (  self      :					Tuple  ,						UpperCamelCase__      :					List[int]  ,						UpperCamelCase__      :					Optional[List[int]] = None  ,						UpperCamelCase__      :					bool = False ):


       """simple docstring"""







       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=UpperCamelCase__  ,						token_ids_a=UpperCamelCase__  ,						already_has_special_tokens=UpperCamelCase__ )

       UpperCamelCase					=  [1] + ([0] * len(UpperCamelCase__ )) + [1]
       if token_ids_a is not None:
          result += ([0] * len(UpperCamelCase__ )) + [1]
       return result






    def  A       (  self      :					int  ,						UpperCamelCase__      :					List[int]  ,						UpperCamelCase__      :					Optional[List[int]] = None ):


       """simple docstring"""







       UpperCamelCase					=  [self.sep_token_id]
       UpperCamelCase					=  [self.cls_token_id]

       UpperCamelCase					=  len(cls + token_ids_a + sep ) * [0]
       if token_ids_a is not None:
          result += len(token_ids_a + sep ) * [1]
       return result






    def  A       (  self      :					Any  ,						UpperCamelCase__      :					str  ,						UpperCamelCase__      :					Optional[str] = None ):


       """simple docstring"""







       return ()






'''simple docstring'''


_lowerCamelCase	:       int      	=	"0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
   from .utils import rich

