import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase__											=					logging.get_logger(__name__)

UpperCamelCase__											=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__											=					{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__											=					{
    'bert-base-uncased': 5_1_2,
    'bert-large-uncased': 5_1_2,
    'bert-base-cased': 5_1_2,
    'bert-large-cased': 5_1_2,
    'bert-base-multilingual-uncased': 5_1_2,
    'bert-base-multilingual-cased': 5_1_2,
    'bert-base-chinese': 5_1_2,
    'bert-base-german-cased': 5_1_2,
    'bert-large-uncased-whole-word-masking': 5_1_2,
    'bert-large-cased-whole-word-masking': 5_1_2,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-base-cased-finetuned-mrpc': 5_1_2,
    'bert-base-german-dbmdz-cased': 5_1_2,
    'bert-base-german-dbmdz-uncased': 5_1_2,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_1_2,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_1_2,
    'wietsedv/bert-base-dutch-cased': 5_1_2,
}

UpperCamelCase__											=					{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}





class      A							(				UpperCAmelCase_     ):
   __UpperCAmelCase : str      =			VOCAB_FILES_NAMES
   __UpperCAmelCase : Optional[Any]      =			PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase : List[Any]      =			PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase : Optional[int]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase : str      =			BertTokenizer


   def __init__(self     :						Any       ,  __UpperCAmelCase     :						Optional[int]=None       ,  __UpperCAmelCase     :						Any=None       ,  __UpperCAmelCase     :						Dict=True       ,  __UpperCAmelCase     :						str="[UNK]"       ,  __UpperCAmelCase     :						List[str]="[SEP]"       ,  __UpperCAmelCase     :						int="[PAD]"       ,  __UpperCAmelCase     :						List[Any]="[CLS]"       ,  __UpperCAmelCase     :						str="[MASK]"       ,  __UpperCAmelCase     :						List[str]=True       ,  __UpperCAmelCase     :						Tuple=None       ,  **__UpperCAmelCase     :						int       ,  )						->				List[Any]:
         """simple docstring"""



         super().__init__(
             __UpperCAmelCase       ,  tokenizer_file=__UpperCAmelCase       ,  do_lower_case=__UpperCAmelCase       ,  unk_token=__UpperCAmelCase       ,  sep_token=__UpperCAmelCase       ,  pad_token=__UpperCAmelCase       ,  cls_token=__UpperCAmelCase       ,  mask_token=__UpperCAmelCase       ,  tokenize_chinese_chars=__UpperCAmelCase       ,  strip_accents=__UpperCAmelCase       ,  **__UpperCAmelCase       ,  )

         UpperCAmelCase__								=  json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             normalizer_state.get("lowercase"       ,  __UpperCAmelCase		) != do_lower_case
             or normalizer_state.get("strip_accents"       ,  __UpperCAmelCase		) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,  __UpperCAmelCase		) != tokenize_chinese_chars
         ):
               UpperCAmelCase__								=  getattr(__UpperCAmelCase       ,  normalizer_state.pop("type"		)		)
               UpperCAmelCase__								=  do_lower_case
               UpperCAmelCase__								=  strip_accents
               UpperCAmelCase__								=  tokenize_chinese_chars
               UpperCAmelCase__								=  normalizer_class(**__UpperCAmelCase		)

         UpperCAmelCase__								=  do_lower_case


   def 			lowercase_      (self     :						int       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						List[str]=None		)						->				Dict:
         """simple docstring"""



         UpperCAmelCase__								=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output


   def 			lowercase_      (self     :						Dict       ,  __UpperCAmelCase     :						List[int]       ,  __UpperCAmelCase     :						Optional[List[int]] = None		)						->				List[int]:
         """simple docstring"""



         UpperCAmelCase__								=  [self.sep_token_id]
         UpperCAmelCase__								=  [self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





   def 			lowercase_      (self     :						Union[str, Any]       ,  __UpperCAmelCase     :						str       ,  __UpperCAmelCase     :						Optional[str] = None		)						->				Tuple[str]:
         """simple docstring"""



         UpperCAmelCase__								=  self._tokenizer.model.save(__UpperCAmelCase       ,  name=__UpperCAmelCase		)
         return tuple(__UpperCAmelCase		)




import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
      from sagemaker import Session, TrainingJobAnalytics
      from sagemaker.huggingface import HuggingFace





@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'		,				'False'     )     ) is not True		,				reason='Skipping test because should only be run when releasing minor transformers version'		,				)
@pytest.mark.usefixtures('sm_env'     )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.g4dn.xlarge',
            'results': {'train_runtime': 6_50, 'eval_accuracy': 0.6, 'eval_loss': 0.9},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.g4dn.xlarge',
            'results': {'train_runtime': 6_00, 'eval_accuracy': 0.3, 'eval_loss': 0.9},
        },
    ]     )
class      A							(				unittest.TestCase     ):


   def 			lowercase_      (self     :						int		)						->				Optional[Any]:
         """simple docstring"""



         if self.framework == "pytorch":
               subprocess.run(
                   f"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()       ,  encoding="utf-8"       ,  check=__UpperCAmelCase       ,  )
         assert hasattr(self       ,  "env"		)


   def 			lowercase_      (self     :						List[Any]       ,  __UpperCAmelCase     :						Optional[int]=1		)						->				Dict:
         """simple docstring"""



         return HuggingFace(
             entry_point=self.script       ,  source_dir=self.env.test_path       ,  role=self.env.role       ,  image_uri=self.env.image_uri       ,  base_job_name=f"""{self.env.base_job_name}-single"""       ,  instance_count=__UpperCAmelCase       ,  instance_type=self.instance_type       ,  debugger_hook_config=__UpperCAmelCase       ,  hyperparameters={**self.env.hyperparameters, "model_name_or_path": self.model_name_or_path}       ,  metric_definitions=self.env.metric_definitions       ,  py_version="py36"       ,  )


   def 			lowercase_      (self     :						Optional[Any]       ,  __UpperCAmelCase     :						Tuple		)						->				Optional[int]:
         """simple docstring"""



         TrainingJobAnalytics(__UpperCAmelCase		).export_csv(f"""{self.env.test_path}/{job_name}_metrics.csv"""		)





   def 			lowercase_      (self     :						Any		)						->				Optional[Any]:
         """simple docstring"""



         UpperCAmelCase__								=  self.create_estimator()

         # run training
         estimator.fit()

         # result dataframe
         UpperCAmelCase__								=  TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

         # extract kpis
         UpperCAmelCase__								=  list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]		)
         UpperCAmelCase__								=  list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]		)
         # get train time from SageMaker job, this includes starting, preprocessing, stopping
         UpperCAmelCase__								=  (
             Session().describe_training_job(estimator.latest_training_job.name		).get("TrainingTimeInSeconds"       ,  9_9_9_9_9_9		)
         )

         # assert kpis
         assert train_runtime <= self.results["train_runtime"]
         assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy		)
         assert all(t <= self.results["eval_loss"] for t in eval_loss		)

         # dump tests result into json file to share in PR
         with open(f"""{estimator.latest_training_job.name}.json"""       ,  "w"		) as outfile:
               json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}       ,  __UpperCAmelCase		)



