


'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase							:   int  						=	logging.get_logger(__name__)
lowerCamelCase							:   List[str]  						=	{'vocab_file': 'spiece.model'}

lowerCamelCase							:   Optional[Any]  						=	{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

lowerCamelCase							:   Any  						=	{
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

lowerCamelCase							:   Optional[Any]  						=	'▁'
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     int  					=							VOCAB_FILES_NAMES
      lowerCAmelCase__					:     Optional[int]  					=							PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__					:     Dict  					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



      def __init__(self    :    Optional[Any]   ,						UpperCamelCase    :    Any   ,						UpperCamelCase    :    List[str]=True   ,						UpperCamelCase    :    Any=True   ,						UpperCamelCase    :    Optional[int]=False   ,						UpperCamelCase    :    List[str]="[CLS]"   ,						UpperCamelCase    :    List[str]="[SEP]"   ,						UpperCamelCase    :    int="<unk>"   ,						UpperCamelCase    :    Optional[Any]="[SEP]"   ,						UpperCamelCase    :    Optional[Any]="<pad>"   ,						UpperCamelCase    :    List[str]="[CLS]"   ,						UpperCamelCase    :    Tuple="[MASK]"   ,						UpperCamelCase    :    Optional[Dict[str, Any]] = None   ,						**UpperCamelCase    :    Dict   ,						):

             '''simple docstring'''







             lowercase__      			= (
                 AddedToken(UpperCamelCase   ,						lstrip=UpperCamelCase   ,						rstrip=UpperCamelCase   ,						normalized=UpperCamelCase )
                 if isinstance(UpperCamelCase   ,						UpperCamelCase )
                 else mask_token
             )

             lowercase__      			= {} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=UpperCamelCase   ,						remove_space=UpperCamelCase   ,						keep_accents=UpperCamelCase   ,						bos_token=UpperCamelCase   ,						eos_token=UpperCamelCase   ,						unk_token=UpperCamelCase   ,						sep_token=UpperCamelCase   ,						pad_token=UpperCamelCase   ,						cls_token=UpperCamelCase   ,						mask_token=UpperCamelCase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**UpperCamelCase   ,						)

             lowercase__      			= do_lower_case
             lowercase__      			= remove_space
             lowercase__      			= keep_accents
             lowercase__      			= vocab_file

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(UpperCamelCase )



      @property
      def       UpperCamelCase__		(self    :    Dict ):

             '''simple docstring'''







             return len(self.sp_model )



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= {self.convert_ids_to_tokens(UpperCamelCase ): i for i in range(self.vocab_size )}
             vocab.update(self.added_tokens_encoder )
             return vocab



      def __getstate__(self    :    int ):

             '''simple docstring'''







             lowercase__      			= self.__dict__.copy()
             lowercase__      			= None
             return state



      def __setstate__(self    :    Tuple   ,						UpperCamelCase    :    List[Any] ):

             '''simple docstring'''







             lowercase__      			= d

             # for backward compatibility
             if not hasattr(self   ,						'''sp_model_kwargs''' ):
                    lowercase__      			= {}

             lowercase__      			= spm.SentencePieceProcessor(**self.sp_model_kwargs )
             self.sp_model.Load(self.vocab_file )



      def       UpperCamelCase__		(self    :    Tuple   ,						UpperCamelCase    :    Dict ):

             '''simple docstring'''







             if self.remove_space:
                    lowercase__      			= ''' '''.join(inputs.strip().split() )
             else:
                    lowercase__      			= inputs
             lowercase__      			= outputs.replace('''``'''   ,						'''"''' ).replace('''\'\''''   ,						'''"''' )

             if not self.keep_accents:
                    lowercase__      			= unicodedata.normalize('''NFKD'''   ,						UpperCamelCase )
                    lowercase__      			= ''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase )] )
             if self.do_lower_case:
                    lowercase__      			= outputs.lower()

             return outputs



      def       UpperCamelCase__		(self    :    Optional[Any]   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= self.preprocess_text(UpperCamelCase )
             lowercase__      			= self.sp_model.encode(UpperCamelCase   ,						out_type=UpperCamelCase )
             lowercase__      			= []
             for piece in pieces:
                    if len(UpperCamelCase ) > 1 and piece[-1] == str(''',''' ) and piece[-2].isdigit():
                           lowercase__      			= self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase   ,						'''''' ) )
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0] ) == 1:
                                         lowercase__      			= cur_pieces[1:]
                                  else:
                                         lowercase__      			= cur_pieces[0][1:]
                           cur_pieces.append(piece[-1] )
                           new_pieces.extend(UpperCamelCase )
                    else:
                           new_pieces.append(UpperCamelCase )

             return new_pieces



      def       UpperCamelCase__		(self    :    str   ,						UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase )



      def       UpperCamelCase__		(self    :    List[str]   ,						UpperCamelCase    :    List[str] ):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase )



      def       UpperCamelCase__		(self    :    str   ,						UpperCamelCase    :    str ):

             '''simple docstring'''







             lowercase__      			= []
             lowercase__      			= ''''''
             lowercase__      			= False
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           if not prev_is_special:
                                  out_string += " "
                           out_string += self.sp_model.decode(UpperCamelCase ) + token
                           lowercase__      			= True
                           lowercase__      			= []
                    else:
                           current_sub_tokens.append(UpperCamelCase )
                           lowercase__      			= False
             out_string += self.sp_model.decode(UpperCamelCase )
             return out_string.strip()



      def       UpperCamelCase__		(self    :    Optional[int]   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [self.cls_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep



      def       UpperCamelCase__		(self    :    int   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None   ,						UpperCamelCase    :    bool = False ):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase   ,						token_ids_a=UpperCamelCase   ,						already_has_special_tokens=UpperCamelCase )

             if token_ids_a is not None:
                    return [1] + ([0] * len(UpperCamelCase )) + [1] + ([0] * len(UpperCamelCase )) + [1]
             return [1] + ([0] * len(UpperCamelCase )) + [1]



      def       UpperCamelCase__		(self    :    Tuple   ,						UpperCamelCase    :    List[int]   ,						UpperCamelCase    :    Optional[List[int]] = None ):

             '''simple docstring'''







             lowercase__      			= [self.sep_token_id]
             lowercase__      			= [self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep ) * [0]
             return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






      def       UpperCamelCase__		(self    :    Optional[Any]   ,						UpperCamelCase    :    str   ,						UpperCamelCase    :    Optional[str] = None ):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase ):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
                    return
             lowercase__      			= os.path.join(
                 UpperCamelCase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

             if os.path.abspath(self.vocab_file ) != os.path.abspath(UpperCamelCase ) and os.path.isfile(self.vocab_file ):
                    copyfile(self.vocab_file   ,						UpperCamelCase )
             elif not os.path.isfile(self.vocab_file ):
                    with open(UpperCamelCase   ,						'''wb''' ) as fi:
                           lowercase__      			= self.sp_model.serialized_model_proto()
                           fi.write(UpperCamelCase )

             return (out_vocab_file,)








import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
    from sagemaker import Session, TrainingJobAnalytics
    from sagemaker.huggingface import HuggingFace







@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"     ,    "False")) is not True     ,    reason="Skipping test because should only be run when releasing minor transformers version"     ,    )
@pytest.mark.usefixtures("sm_env")
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 650, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "pytorch",
            "script": "run_ddp.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "tensorflow",
            "script": "run_tf_dist.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.6, "eval_loss": 0.7},
        },
    ])
class      lowerCAmelCase__       (				unittest.TestCase):




     '''simple docstring'''


     def  _lowerCamelCase (						self)			->				str:
            if self.framework == "pytorch":
                   subprocess.run(
                       F"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split() ,						encoding="utf-8" ,						check=__lowerCamelCase ,						)
            assert hasattr(self ,						"env")


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Tuple:
            _A      :       Dict				    = F"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
            # distributed data settings
            _A      :       Optional[Any]				    = {"smdistributed": {"dataparallel": {"enabled": True}}} if self.script != "run_ddp.py" else None

            # creates estimator
            return HuggingFace(
                entry_point=self.script ,						source_dir=self.env.test_path ,						role=self.env.role ,						image_uri=self.env.image_uri ,						base_job_name=__lowerCamelCase ,						instance_count=__lowerCamelCase ,						instance_type=self.instance_type ,						debugger_hook_config=__lowerCamelCase ,						hyperparameters={**self.env.distributed_hyperparameters, "model_name_or_path": self.model_name_or_path} ,						metric_definitions=self.env.metric_definitions ,						distribution=__lowerCamelCase ,						py_version="py36" ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Optional[Any]:
            TrainingJobAnalytics(__lowerCamelCase).export_csv(F"{self.env.test_path}/{job_name}_metrics.csv")


     @parameterized.expand([(2,)])
     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Any:
            # create estimator
            _A      :       Union[str, Any]				    = self.create_estimator(__lowerCamelCase)

            # run training
            estimator.fit()

            # result dataframe
            _A      :       Optional[Any]				    = TrainingJobAnalytics(estimator.latest_training_job.name).dataframe()

            # extract kpis
            _A      :       List[Any]				    = list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"])
            _A      :       Dict				    = list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"])
            # get train time from SageMaker job, this includes starting, preprocessing, stopping
            _A      :       Optional[Any]				    = (
                Session().describe_training_job(estimator.latest_training_job.name).get("TrainingTimeInSeconds" ,						9_9_9_9_9_9)
            )

            # assert kpis
            assert train_runtime <= self.results["train_runtime"]
            assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy)
            assert all(t <= self.results["eval_loss"] for t in eval_loss)

            # dump tests result into json file to share in PR
            with open(F"{estimator.latest_training_job.name}.json" ,						"w") as outfile:
                   json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss} ,						__lowerCamelCase)
