





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


__A									=				logging.get_logger(__name__)
__A									=				{"vocab_file": "spiece.model"}

__A									=				{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

__A									=				{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			VOCAB_FILES_NAMES
 lowercase_		=			PRETRAINED_VOCAB_FILES_MAP
 lowercase_		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_		=			["input_ids", "attention_mask"]

 def __init__(self				:   List[str]       , UpperCAmelCase_				:   List[Any]       , UpperCAmelCase_				:   List[Any]=False       , UpperCAmelCase_				:   str=False       , UpperCAmelCase_				:   Tuple=False       , UpperCAmelCase_				:   Tuple=None       , UpperCAmelCase_				:   Any=None       , UpperCAmelCase_				:   Optional[Any]=None       , UpperCAmelCase_				:   int=None       , UpperCAmelCase_				:   Optional[Dict[str, Any]] = None       , **UpperCAmelCase_				:   List[Any]       , )       ->None:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  ={} if sp_model_kwargs is None else sp_model_kwargs

   lowerCamelCase__:      Tuple			  =kwargs.get("name_or_path")
   if name_or_path is None:
     logger.warning(
         "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
         " you are testing the model, this can safely be ignored")
     lowerCamelCase__:      List[str]			  ="None"

   # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
   lowerCamelCase__:      str			  ="<|endoftext|>" if eos_token is None else eos_token
   lowerCamelCase__:      List[str]			  ="<unk>" if unk_token is None else unk_token
   if "gpt-sw3-7b" in name_or_path:
     lowerCamelCase__:      Optional[Any]			  =unk_token if pad_token is None else pad_token
     lowerCamelCase__:      Tuple			  =eos_token if bos_token is None else bos_token
   else:
     lowerCamelCase__:      List[Any]			  ="<pad>" if pad_token is None else pad_token
     lowerCamelCase__:      Union[str, Any]			  ="<s>" if bos_token is None else bos_token

   super().__init__(
       do_lower_case=UpperCAmelCase_       , remove_space=UpperCAmelCase_       , keep_accents=UpperCAmelCase_       , bos_token=UpperCAmelCase_       , eos_token=UpperCAmelCase_       , unk_token=UpperCAmelCase_       , pad_token=UpperCAmelCase_       , sp_model_kwargs=self.sp_model_kwargs       , **UpperCAmelCase_       , )

   lowerCamelCase__:      Dict			  =do_lower_case
   lowerCamelCase__:      Optional[Any]			  =remove_space
   lowerCamelCase__:      Dict			  =keep_accents
   lowerCamelCase__:      str			  =vocab_file

   lowerCamelCase__:      str			  =spm.SentencePieceProcessor(**self.sp_model_kwargs)
   self.sp_model.Load(UpperCAmelCase_)

   # Used for whitespace normalization in input texts
   # fmt : off
   lowerCamelCase__:      Union[str, Any]			  ={" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
   # fmt : on

   # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
   lowerCamelCase__:      Optional[int]			  =re.compile(
       F"""[{"".join(map(UpperCAmelCase_       , list(range(0       , 9)) + list(range(11       , 32)) + list(range(127       , 160)) + [160, 173, 8_203]))}]""")

 def __getstate__(self				:   List[str])       ->str:



   '''simple docstring'''



   lowerCamelCase__:      str			  =self.__dict__.copy()
   lowerCamelCase__:      List[Any]			  =None
   return state

 def __setstate__(self				:   Any       , UpperCAmelCase_				:   Optional[int])       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Tuple			  =d

   # for backward compatibility
   if not hasattr(self       , "sp_model_kwargs"):
     lowerCamelCase__:      Optional[int]			  ={}

   lowerCamelCase__:      List[str]			  =spm.SentencePieceProcessor(**self.sp_model_kwargs)
   self.sp_model.Load(self.vocab_file)

 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->int:



   '''simple docstring'''



   return len(self.sp_model)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict       , UpperCAmelCase_				:   str)       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Optional[Any]			  =self.non_printing_characters_re.sub(""       , UpperCAmelCase_)

   # Normalize whitespaces
   lowerCamelCase__:      int			  ="".join([char if char not in self.whitespaces else " " for char in text])

   # NFC Unicode normalization
   lowerCamelCase__:      Optional[Any]			  =unicodedata.normalize("NFC"       , UpperCAmelCase_)
   return text

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict       , UpperCAmelCase_				:   str       , **UpperCAmelCase_				:   Any)       ->List[str]:



   '''simple docstring'''



   lowerCamelCase__:      Tuple			  =self.preprocess_text(UpperCAmelCase_)
   return self.sp_model.encode(UpperCAmelCase_       , out_type=UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict       , UpperCAmelCase_				:   str)       ->int:



   '''simple docstring'''



   return self.sp_model.PieceToId(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple       , UpperCAmelCase_				:   int)       ->str:



   '''simple docstring'''



   return self.sp_model.IdToPiece(UpperCAmelCase_)

 @staticmethod
 def 							SCREAMING_SNAKE_CASE_					(UpperCAmelCase_				:   str)       ->str:



   '''simple docstring'''



   return out_string

 def 							SCREAMING_SNAKE_CASE_					(self				:   str       , UpperCAmelCase_				:   List[str])       ->str:



   '''simple docstring'''



   lowerCamelCase__:      Dict			  =[]
   lowerCamelCase__:      Union[str, Any]			  =""
   lowerCamelCase__:      Any			  =False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
       if not prev_is_special:
         out_string += " "

       out_string += self.sp_model.decode(UpperCAmelCase_) + token
       lowerCamelCase__:      List[Any]			  =True
       lowerCamelCase__:      Tuple			  =[]
     else:
       current_sub_tokens.append(UpperCAmelCase_)
       lowerCamelCase__:      str			  =False
   out_string += self.sp_model.decode(UpperCAmelCase_)

   return out_string

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[str])       ->Dict[str, int]:



   '''simple docstring'''



   lowerCamelCase__:      List[str]			  ={self.convert_ids_to_tokens(UpperCAmelCase_): i for i in range(self.vocab_size)}
   vocab.update(self.added_tokens_encoder)
   return vocab

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict       , UpperCAmelCase_				:   str       , UpperCAmelCase_				:   Optional[str] = None)       ->Tuple[str]:



   '''simple docstring'''



   if not os.path.isdir(UpperCAmelCase_):
     logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
     return
   lowerCamelCase__:      int			  =os.path.join(
       UpperCAmelCase_       , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

   if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase_) and os.path.isfile(self.vocab_file):
     copyfile(self.vocab_file       , UpperCAmelCase_)
   elif not os.path.isfile(self.vocab_file):
     with open(UpperCAmelCase_       , "wb") as fi:
       lowerCamelCase__:      Any			  =self.sp_model.serialized_model_proto()
       fi.write(UpperCAmelCase_)

   return (out_vocab_file,)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict       , UpperCAmelCase_				:   Union[str, List[str]]       , UpperCAmelCase_				:   Union[str, bool] = False)       ->Union[List[int], List[List[int]], "torch.Tensor"]:



   '''simple docstring'''



   if isinstance(UpperCAmelCase_       , UpperCAmelCase_):
     lowerCamelCase__:      int			  =self.preprocess_text(UpperCAmelCase_)
     lowerCamelCase__:      Optional[Any]			  =self.sp_model.encode(UpperCAmelCase_)
   else:
     lowerCamelCase__:      Any			  =[self.preprocess_text(UpperCAmelCase_) for t in text]
     lowerCamelCase__:      Tuple			  =self.sp_model.encode(UpperCAmelCase_)

   if return_tensors is True or return_tensors == "pt":
     lowerCamelCase__:      List[str]			  =torch.tensor(UpperCAmelCase_)

   return token_ids

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple       , UpperCAmelCase_				:   Union[int, List[int]])       ->str:



   '''simple docstring'''



   return self.sp_model.decode(UpperCAmelCase_)







 def 							SCREAMING_SNAKE_CASE_					(self				:   int       , UpperCAmelCase_				:   "Conversation")       ->List[int]:



   '''simple docstring'''



   lowerCamelCase__:      List[Any]			  =[F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
   lowerCamelCase__:      List[str]			  =(
       F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(UpperCAmelCase_) + F"""{self.bos_token}Bot:"""
   )
   return self.encode(text=UpperCAmelCase_)








import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate



@dataclass(frozen=__SCREAMING_SNAKE_CASE    )
class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			field(default="image-classification"    ,							metadata={"include_in_asdict_even_if_is_default": True}    )
 lowercase_		=			Features({"image": Image()}    )
 lowercase_		=			Features({"labels": ClassLabel}    )
 lowercase_		=			"image"
 lowercase_		=			"labels"

 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple       , UpperCAmelCase_				:   Union[str, Any])       ->Tuple:



   '''simple docstring'''



   if self.label_column not in features:
     raise ValueError(F"""Column {self.label_column} is not present in features.""")
   if not isinstance(features[self.label_column]       , UpperCAmelCase_):
     raise ValueError(F"""Column {self.label_column} is not a ClassLabel.""")
   lowerCamelCase__:      List[Any]			  =copy.deepcopy(self)
   lowerCamelCase__:      Optional[int]			  =self.label_schema.copy()
   lowerCamelCase__:      int			  =features[self.label_column]
   lowerCamelCase__:      int			  =label_schema
   return task_template







 @property
 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Dict[str, str]:



   '''simple docstring'''



   return {
       self.image_column: "image",
       self.label_column: "labels",
   }

