


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_      : str  					=       logging.get_logger(__name__)


UpperCAmelCase_      : List[Any]  					=       {
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
UpperCAmelCase_      : Union[str, Any]  					=       111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
UpperCAmelCase_      : List[str]  					=       0

UpperCAmelCase_      : str  					=       0Xe_000
UpperCAmelCase_      : Optional[Any]  					=       0Xe_001
UpperCAmelCase_      : Union[str, Any]  					=       0Xe_002
UpperCAmelCase_      : Tuple  					=       0Xe_003
UpperCAmelCase_      : List[Any]  					=       0Xe_004

# Maps special codepoints to human-readable names.
UpperCAmelCase_      : Dict[int, str]  					=       {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
UpperCAmelCase_      : Dict[str, int]  					=       {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      snake_case__				:     int        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      def __init__(       self				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       Tuple=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       Any=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       str=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       int=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       int=chr(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__				:       List[str]=False     ,							SCREAMING_SNAKE_CASE__				:       List[str]=2_0_4_8     ,							**SCREAMING_SNAKE_CASE__				:       List[Any]     ,							)		->			List[Any]:
             a_					:		List[str]							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else bos_token
             a_					:		Optional[Any]							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else eos_token
             a_					:		Optional[int]							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else sep_token
             a_					:		Union[str, Any]							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else cls_token
             a_					:		Tuple							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             a_					:		Optional[Any]							=  AddedToken(SCREAMING_SNAKE_CASE__     ,							lstrip=SCREAMING_SNAKE_CASE__     ,							rstrip=SCREAMING_SNAKE_CASE__     ) if isinstance(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ) else mask_token

             super().__init__(
                 bos_token=SCREAMING_SNAKE_CASE__     ,							eos_token=SCREAMING_SNAKE_CASE__     ,							sep_token=SCREAMING_SNAKE_CASE__     ,							cls_token=SCREAMING_SNAKE_CASE__     ,							pad_token=SCREAMING_SNAKE_CASE__     ,							mask_token=SCREAMING_SNAKE_CASE__     ,							add_prefix_space=SCREAMING_SNAKE_CASE__     ,							model_max_length=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     ,							)

             # Creates a mapping for looking up the IDs of special symbols.
             a_					:		Dict[str, int]							=  {}
             for codepoint, name in SPECIAL_CODEPOINTS.items():
                    a_					:		Optional[int]							=  codepoint

             # Creates a mapping for looking up the string forms of special symbol IDs.
             a_					:		Dict[int, str]							=  {
                 codepoint: name for name, codepoint in self._special_codepoints.items()
             }

             a_					:		Dict							=  UNICODE_VOCAB_SIZE
             a_					:		Any							=  len(self._special_codepoints     )
      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			int:
             return self._unicode_vocab_size
      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     ,							SCREAMING_SNAKE_CASE__				:       str     )		->			List[str]:
             return list(SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       int     ,							SCREAMING_SNAKE_CASE__				:       str     )		->			int:
             try:
                    return ord(SCREAMING_SNAKE_CASE__     )
             except TypeError:
                    raise ValueError(F"""invalid token: '{token}'"""     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Any     ,							SCREAMING_SNAKE_CASE__				:       int     )		->			str:
             try:
                    if index in SPECIAL_CODEPOINTS:
                           return SPECIAL_CODEPOINTS[index]
                    return chr(SCREAMING_SNAKE_CASE__     )
             except TypeError:
                    raise ValueError(F"""invalid id: {index}"""     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     )		->			Union[str, Any]:
             return "".join(SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       List[int]     ,							SCREAMING_SNAKE_CASE__				:       Optional[List[int]] = None     )		->			List[int]:
             a_					:		int							=  [self.sep_token_id]
             a_					:		Optional[int]							=  [self.cls_token_id]

             a_					:		Optional[int]							=  cls + token_ids_a + sep
             if token_ids_a is not None:
                    result += token_ids_a + sep
             return result
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       List[int]     ,							SCREAMING_SNAKE_CASE__				:       Optional[List[int]] = None     ,							SCREAMING_SNAKE_CASE__				:       bool = False     )		->			List[int]:
             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=SCREAMING_SNAKE_CASE__     ,							token_ids_a=SCREAMING_SNAKE_CASE__     ,							already_has_special_tokens=SCREAMING_SNAKE_CASE__     )

             a_					:		Dict							=  [1] + ([0] * len(SCREAMING_SNAKE_CASE__     )) + [1]
             if token_ids_a is not None:
                    result += ([0] * len(SCREAMING_SNAKE_CASE__     )) + [1]
             return result
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     ,							SCREAMING_SNAKE_CASE__				:       List[int]     ,							SCREAMING_SNAKE_CASE__				:       Optional[List[int]] = None     )		->			List[int]:
             a_					:		int							=  [self.sep_token_id]
             a_					:		int							=  [self.cls_token_id]

             a_					:		Optional[Any]							=  len(cls + token_ids_a + sep     ) * [0]
             if token_ids_a is not None:
                    result += len(token_ids_a + sep     ) * [1]
             return result

      def        SCREAMING_SNAKE_CASE							(       self				:       int     ,							SCREAMING_SNAKE_CASE__				:       str     ,							SCREAMING_SNAKE_CASE__				:       Optional[str] = None     )		->			Union[str, Any]:
             return ()






from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:NestedDataStructureLike[PathLike]			,			lowerCAmelCase__			:Optional[NamedSplit] = None			,			lowerCAmelCase__			:Optional[Features] = None			,			lowerCAmelCase__			:str = None			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:Optional[int] = None			,			**lowerCAmelCase__			:Optional[int]			,			)		->			Tuple:
										super().__init__(
										    lowerCAmelCase__			,			split=lowerCAmelCase__			,			features=lowerCAmelCase__			,			cache_dir=lowerCAmelCase__			,			keep_in_memory=lowerCAmelCase__			,			streaming=lowerCAmelCase__			,			num_proc=lowerCAmelCase__			,			**lowerCAmelCase__			,			)
										__SCREAMING_SNAKE_CASE						:     List[str]					=		path_or_paths if isinstance(lowerCAmelCase__			,			lowerCAmelCase__	) else {self.split: path_or_paths}
										__SCREAMING_SNAKE_CASE						:     int					=		Text(
										    cache_dir=lowerCAmelCase__			,			data_files=lowerCAmelCase__			,			features=lowerCAmelCase__			,			**lowerCAmelCase__			,			)



				def 		__magic_name__(       self			:Dict	)		->			Tuple:
										# Build iterable dataset
										if self.streaming:
																__SCREAMING_SNAKE_CASE						:     int					=		self.builder.as_streaming_dataset(split=self.split	)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		None
																__SCREAMING_SNAKE_CASE						:     str					=		None
																__SCREAMING_SNAKE_CASE						:     Dict					=		None
																__SCREAMING_SNAKE_CASE						:     Tuple					=		None

																self.builder.download_and_prepare(
																    download_config=lowerCAmelCase__			,			download_mode=lowerCAmelCase__			,			verification_mode=lowerCAmelCase__			,			base_path=lowerCAmelCase__			,			num_proc=self.num_proc			,			)
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.builder.as_dataset(
																    split=self.split			,			verification_mode=lowerCAmelCase__			,			in_memory=self.keep_in_memory	)
										return dataset
