


'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


a_  :				Union[str, Any]						= logging.get_logger(__name__)
a_  :				int						= {"""vocab_file""": """spiece.model"""}

a_  :				Tuple						= {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

a_  :				Tuple						= {
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  VOCAB_FILES_NAMES
  _lowerCamelCase		    =  PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase		    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase		    =  ["input_ids", "attention_mask"]






  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase=False			,  UpperCamelCase=False			,  UpperCamelCase=False			,  UpperCamelCase=None			,  UpperCamelCase=None			,  UpperCamelCase=None			,  UpperCamelCase=None			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""
   lowerCamelCase_      =  {} if sp_model_kwargs is None else sp_model_kwargs

   lowerCamelCase_      =  kwargs.get("name_or_path"     )
   if name_or_path is None:
    logger.warning(
        "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
        " you are testing the model, this can safely be ignored"     )
    lowerCamelCase_      =  "None"

   # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
   lowerCamelCase_      =  "<|endoftext|>" if eos_token is None else eos_token
   lowerCamelCase_      =  "<unk>" if unk_token is None else unk_token
   if "gpt-sw3-7b" in name_or_path:
    lowerCamelCase_      =  unk_token if pad_token is None else pad_token
    lowerCamelCase_      =  eos_token if bos_token is None else bos_token
   else:
    lowerCamelCase_      =  "<pad>" if pad_token is None else pad_token
    lowerCamelCase_      =  "<s>" if bos_token is None else bos_token

   super().__init__(
       do_lower_case=UpperCamelCase			,  remove_space=UpperCamelCase			,  keep_accents=UpperCamelCase			,  bos_token=UpperCamelCase			,  eos_token=UpperCamelCase			,  unk_token=UpperCamelCase			,  pad_token=UpperCamelCase			,  sp_model_kwargs=self.sp_model_kwargs			,  **UpperCamelCase			,  )

   lowerCamelCase_      =  do_lower_case
   lowerCamelCase_      =  remove_space
   lowerCamelCase_      =  keep_accents
   lowerCamelCase_      =  vocab_file

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(UpperCamelCase     )

   # Used for whitespace normalization in input texts
   # fmt : off
   lowerCamelCase_      =  {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
   # fmt : on

   # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
   lowerCamelCase_      =  re.compile(
       f'''[{"".join(map(UpperCamelCase			,  list(range(0			,  9     )     ) + list(range(11			,  32     )     ) + list(range(127			,  160     )     ) + [160, 173, 8203]     )     )}]'''     )






  def __getstate__(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.__dict__.copy()
   lowerCamelCase_      =  None
   return state






  def __setstate__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  d

   # for backward compatibility
   if not hasattr(self			,  "sp_model_kwargs"     ):
    lowerCamelCase_      =  {}

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )






  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def   snake_case		(    self     ):







   """simple docstring"""
   return len(self.sp_model     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.non_printing_characters_re.sub(""			,  UpperCamelCase     )

   # Normalize whitespaces
   lowerCamelCase_      =  "".join([char if char not in self.whitespaces else " " for char in text]     )

   # NFC Unicode normalization
   lowerCamelCase_      =  unicodedata.normalize("NFC"			,  UpperCamelCase     )
   return text






  def   snake_case		(    self			,  UpperCamelCase			,  **UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.preprocess_text(UpperCamelCase     )
   return self.sp_model.encode(UpperCamelCase			,  out_type=UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.PieceToId(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.IdToPiece(UpperCamelCase     )






  @staticmethod
  def   snake_case		(    UpperCamelCase     ):







   """simple docstring"""
   return out_string






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   lowerCamelCase_      =  ""
   lowerCamelCase_      =  False
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
     if not prev_is_special:
      out_string += " "

     out_string += self.sp_model.decode(UpperCamelCase     ) + token
     lowerCamelCase_      =  True
     lowerCamelCase_      =  []
    else:
     current_sub_tokens.append(UpperCamelCase     )
     lowerCamelCase_      =  False
   out_string += self.sp_model.decode(UpperCamelCase     )

   return out_string






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  {self.convert_ids_to_tokens(UpperCamelCase     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   if not os.path.isdir(UpperCamelCase     ):
    logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
    return
   lowerCamelCase_      =  os.path.join(
       UpperCamelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase     ) and os.path.isfile(self.vocab_file     ):
    copyfile(self.vocab_file			,  UpperCamelCase     )
   elif not os.path.isfile(self.vocab_file     ):
    with open(UpperCamelCase			,  "wb"     ) as fi:
     lowerCamelCase_      =  self.sp_model.serialized_model_proto()
     fi.write(UpperCamelCase     )

   return (out_vocab_file,)






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = False     ):







   """simple docstring"""

   if isinstance(UpperCamelCase			,  UpperCamelCase     ):
    lowerCamelCase_      =  self.preprocess_text(UpperCamelCase     )
    lowerCamelCase_      =  self.sp_model.encode(UpperCamelCase     )
   else:
    lowerCamelCase_      =  [self.preprocess_text(UpperCamelCase     ) for t in text]
    lowerCamelCase_      =  self.sp_model.encode(UpperCamelCase     )

   if return_tensors is True or return_tensors == "pt":
    lowerCamelCase_      =  torch.tensor(UpperCamelCase     )

   return token_ids






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.decode(UpperCamelCase     )







  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
   lowerCamelCase_      =  (
       f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(UpperCamelCase     ) + f'''{self.bos_token}Bot:'''
   )
   return self.encode(text=UpperCamelCase     )







'''simple docstring'''

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser





def __snake_case					(      ):
 lowerCamelCase_      =  ArgumentParser("Accelerate CLI tool"					,				usage="accelerate <command> [<args>]"					,				allow_abbrev=UpperCAmelCase_		)
 lowerCamelCase_      =  parser.add_subparsers(help="accelerate command helpers"		)

 # Register commands
 get_config_parser(subparsers=UpperCAmelCase_		)
 env_command_parser(subparsers=UpperCAmelCase_		)
 launch_command_parser(subparsers=UpperCAmelCase_		)
 tpu_command_parser(subparsers=UpperCAmelCase_		)
 test_command_parser(subparsers=UpperCAmelCase_		)

 # Let's go
 lowerCamelCase_      =  parser.parse_args()

 if not hasattr(UpperCAmelCase_					,				"func"		):
  parser.print_help()
  exit(1		)

 # Run
 args.func(UpperCAmelCase_		)


if __name__ == "__main__":
     main()



