


'''simple docstring'''



import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


__A		=logging.get_logger(__name__)

__A		={'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

__A		={
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

__A		={
    'abeja/gpt-neox-japanese-2.7b': 20_48,
}







def 			_UpperCamelCase					(						UpperCamelCase__       ,					UpperCamelCase__    ):
			with open(UpperCamelCase__       ,					"""r"""       ,					encoding="""utf-8"""    ) as f:
						UpperCAmelCase__     :    Tuple         =   json.loads(f.read()    )

			UpperCAmelCase__     :    str         =   collections.OrderedDict()
			UpperCAmelCase__     :    Union[str, Any]         =   collections.OrderedDict()
			UpperCAmelCase__     :    Union[str, Any]         =   collections.OrderedDict()
			with open(UpperCamelCase__       ,					"""r"""       ,					encoding="""utf-8"""    ) as f:
						UpperCAmelCase__     :    Any         =   f.readlines()
			UpperCAmelCase__     :    List[Any]         =   [[t.rstrip("""\n"""    )] if (t == "," or "," not in t) else t.rstrip("""\n"""    ).split(""","""    ) for t in token]
			for idx, b in enumerate(UpperCamelCase__    ):
						UpperCAmelCase__     :    Optional[Any]         =   b
						UpperCAmelCase__     :    List[str]         =   idx
						for wd in b:
									UpperCAmelCase__     :    Union[str, Any]         =   idx

			return vocab, raw_vocab, ids_to_tokens, emoji


class 						_snake_case    (   a__		):
					lowerCAmelCase					:int            =    VOCAB_FILES_NAMES
					lowerCAmelCase					:Union[str, Any]            =    PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase					:Dict            =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase					:Optional[int]            =    ['''input_ids''', '''attention_mask''']




					def __init__(					self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase="<|endoftext|>"							,							_lowerCamelCase="<|endoftext|>"							,							_lowerCamelCase="<|startoftext|>"							,							_lowerCamelCase="<|endoftext|>"							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								super().__init__(
								    unk_token=__lowerCamelCase							,							pad_token=__lowerCamelCase							,							bos_token=__lowerCamelCase							,							eos_token=__lowerCamelCase							,							do_clean_text=__lowerCamelCase							,							**__lowerCamelCase							,							)
								if not os.path.isfile(__lowerCamelCase):
											raise ValueError(
											    f'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
											    """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`""")
								if not os.path.isfile(__lowerCamelCase):
											raise ValueError(
											    f'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
											    """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`""")
								UpperCAmelCase__     :    Optional[int]         =   do_clean_text
								UpperCAmelCase__     :    Any         =   load_vocab_and_emoji(__lowerCamelCase							,							__lowerCamelCase)
								UpperCAmelCase__     :    Union[str, Any]         =   SubWordJapaneseTokenizer(
								    vocab=self.vocab							,							ids_to_tokens=self.ids_to_tokens							,							emoji=self.emoji)




					@property
					def  snake_case__  (					self):
								# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
								return len(self.raw_vocab)




					def  snake_case__  (					self):
								return dict(self.raw_vocab							,							**self.added_tokens_encoder)




					def  snake_case__  (					self							,							_lowerCamelCase):
								return self.subword_tokenizer.tokenize(__lowerCamelCase							,							clean=self.do_clean_text)




					def  snake_case__  (					self							,							_lowerCamelCase):
								return self.vocab.get(__lowerCamelCase							,							self.vocab.get(self.unk_token))




					def  snake_case__  (					self							,							_lowerCamelCase):
								return self.subword_tokenizer.convert_id_to_token(__lowerCamelCase)




					def  snake_case__  (					self							,							_lowerCamelCase):
								UpperCAmelCase__     :    List[str]         =   "".join(__lowerCamelCase).strip()
								return out_string




					def  snake_case__  (					self							,							_lowerCamelCase):
								UpperCAmelCase__     :    Optional[Any]         =   []
								for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(__lowerCamelCase							,							add_special_tokens=__lowerCamelCase) + [self.eos_token_id])

								if len(__lowerCamelCase) > self.model_max_length:
											UpperCAmelCase__     :    List[Any]         =   input_ids[-self.model_max_length :]
								return input_ids




					def  snake_case__  (					self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__     :    Optional[Any]         =   0
								if os.path.isdir(__lowerCamelCase):
											UpperCAmelCase__     :    Optional[int]         =   os.path.join(
											    __lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
											UpperCAmelCase__     :    List[str]         =   os.path.join(
											    __lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""])
								else:
											UpperCAmelCase__     :    Union[str, Any]         =   (
											    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
											)
											UpperCAmelCase__     :    Dict         =   (
											    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
											)
								with open(__lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as writer:
											for token_index, token in self.ids_to_tokens.items():
														if index != token_index:
																	logger.warning(
																	    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																	    """ Please check that the vocabulary is not corrupted!""")
																	UpperCAmelCase__     :    List[Any]         =   token_index
														writer.write(""",""".join(__lowerCamelCase) + """\n""")
														index += 1
								with open(__lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as writer:
											json.dump(self.emoji							,							__lowerCamelCase)
								return vocab_file, emoji_file


class 						_snake_case    (   a__		):




					def __init__(					self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__     :    Optional[int]         =   vocab  # same as swe
								UpperCAmelCase__     :    Optional[Any]         =   ids_to_tokens  # same as bpe
								UpperCAmelCase__     :    Tuple         =   emoji
								UpperCAmelCase__     :    Dict         =   np.max([len(__lowerCamelCase) for w in self.vocab.keys()])
								UpperCAmelCase__     :    int         =   re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)""")
								UpperCAmelCase__     :    int         =   re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*""")
								UpperCAmelCase__     :    str         =   re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}""")
								UpperCAmelCase__     :    Any         =   re.compile(
								    r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*""")
								UpperCAmelCase__     :    List[Any]         =   re.compile(
								    r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*""")
								UpperCAmelCase__     :    Optional[Any]         =   re.compile(
								    r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*""")
								UpperCAmelCase__     :    Tuple         =   "─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
								UpperCAmelCase__     :    Dict         =   "▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
								UpperCAmelCase__     :    str         =   str.maketrans({k: """<BLOCK>""" for k in keisen + blocks})




					def __len__(					self):
								return len(self.ids_to_tokens)




					def  snake_case__  (					self							,							_lowerCamelCase):
								UpperCAmelCase__     :    Optional[Any]         =   self.content_repattera.sub("""<URL>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    Dict         =   self.content_repattera.sub("""<EMAIL>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    Optional[Any]         =   self.content_repattera.sub("""<TEL>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    Optional[int]         =   self.content_repattera.sub("""<DATE>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    List[str]         =   self.content_repattera.sub("""<DATE>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    List[str]         =   self.content_repattera.sub("""<PRICE>"""							,							__lowerCamelCase)
								UpperCAmelCase__     :    int         =   content.translate(self.content_transa)
								while "<BLOCK><BLOCK>" in content:
											UpperCAmelCase__     :    List[Any]         =   content.replace("""<BLOCK><BLOCK>"""							,							"""<BLOCK>""")
								return content




					def  snake_case__  (					self							,							_lowerCamelCase							,							_lowerCamelCase=False):
								UpperCAmelCase__     :    Tuple         =   text.replace(""" """							,							"""<SP>""")
								UpperCAmelCase__     :    Tuple         =   text.replace("""　"""							,							"""<SP>""")
								UpperCAmelCase__     :    List[Any]         =   text.replace("""\r\n"""							,							"""<BR>""")
								UpperCAmelCase__     :    Any         =   text.replace("""\n"""							,							"""<BR>""")
								UpperCAmelCase__     :    Optional[Any]         =   text.replace("""\r"""							,							"""<BR>""")
								UpperCAmelCase__     :    str         =   text.replace("""\t"""							,							"""<TAB>""")
								UpperCAmelCase__     :    List[Any]         =   text.replace("""—"""							,							"""ー""")
								UpperCAmelCase__     :    List[Any]         =   text.replace("""−"""							,							"""ー""")
								for k, v in self.emoji["emoji"].items():
											if k in text:
														UpperCAmelCase__     :    Tuple         =   text.replace(__lowerCamelCase							,							__lowerCamelCase)
								if clean:
											UpperCAmelCase__     :    Union[str, Any]         =   self.clean_text(__lowerCamelCase)

								def check_simbol(_lowerCamelCase):
											UpperCAmelCase__     :    Optional[int]         =   x.encode()
											if len(__lowerCamelCase) == 1 and len(__lowerCamelCase) == 2:
														UpperCAmelCase__     :    int         =   (int(e[0]) << 8) + int(e[1])
														if (
														    (c >= 0Xc_2_a_1 and c <= 0Xc_2_b_f)
														    or (c >= 0Xc_7_8_0 and c <= 0Xc_7_8_3)
														    or (c >= 0Xc_a_b_9 and c <= 0Xc_b_b_f)
														    or (c >= 0Xc_c_8_0 and c <= 0Xc_d_a_2)
														):
																	return True
											return False

								def checkuae(_lowerCamelCase):
											UpperCAmelCase__     :    Optional[Any]         =   x.encode()
											if len(__lowerCamelCase) == 1 and len(__lowerCamelCase) == 3:
														UpperCAmelCase__     :    Any         =   (int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
														if c >= 0Xe_2_8_0_8_0 and c <= 0Xe_2_b_0_7_f:
																	return True
											return False

								UpperCAmelCase__     :    Tuple         =   0
								UpperCAmelCase__     :    str         =   []
								while pos < len(__lowerCamelCase):
											UpperCAmelCase__     :    List[str]         =   min(len(__lowerCamelCase)							,							pos + self.maxlen + 1) if text[pos] == "<" else pos + 3
											UpperCAmelCase__     :    Tuple         =   []  # (token_id, token, pos)
											for e in range(__lowerCamelCase							,							__lowerCamelCase							,							-1):
														UpperCAmelCase__     :    Tuple         =   text[pos:e]
														if wd in self.vocab:
																	if wd[0] == "<" and len(__lowerCamelCase) > 2:
																				UpperCAmelCase__     :    str         =   [(self.vocab[wd], wd, e)]
																				break
																	else:
																				candidates.append((self.vocab[wd], wd, e))
											if len(__lowerCamelCase) > 0:
														# the smallest token_id is adopted
														UpperCAmelCase__     :    Tuple         =   sorted(__lowerCamelCase							,							key=lambda _lowerCamelCase: x[0])[0]
														result.append(__lowerCamelCase)
														UpperCAmelCase__     :    Any         =   e
											else:
														UpperCAmelCase__     :    Union[str, Any]         =   pos + 1
														UpperCAmelCase__     :    Any         =   text[pos:end]
														if check_simbol(__lowerCamelCase):
																	result.append("""<KIGOU>""")
														elif checkuae(__lowerCamelCase):
																	result.append("""<U2000U2BFF>""")
														else:
																	for i in wd.encode("""utf-8"""):
																				result.append("""<|byte%d|>""" % i)
														UpperCAmelCase__     :    Tuple         =   end
								return result




					def  snake_case__  (					self							,							_lowerCamelCase							,							_lowerCamelCase="\n"):
								UpperCAmelCase__     :    Union[str, Any]         =   []
								UpperCAmelCase__     :    int         =   []
								UpperCAmelCase__     :    List[str]         =   self.ids_to_tokens[index][0]
								if word[:6] == "<|byte" and word[-2:] == "|>":
											byte_tokens.append(int(word[6:-2]))
								else:
											if len(__lowerCamelCase) > 0:
														words.append(bytearray(__lowerCamelCase).decode("""utf-8"""							,							errors="""replace"""))
														UpperCAmelCase__     :    str         =   []
											if word[:7] == "<|emoji" and word[-2:] == "|>":
														words.append(self.emoji["""emoji_inv"""][word])
											elif word == "<SP>":
														words.append(""" """)
											elif word == "<BR>":
														words.append(__lowerCamelCase)
											elif word == "<TAB>":
														words.append("""\t""")
											elif word == "<BLOCK>":
														words.append("""▀""")
											elif word == "<KIGOU>":
														words.append("""ǀ""")
											elif word == "<U2000U2BFF>":
														words.append("""‖""")
											else:
														words.append(__lowerCamelCase)
								if len(__lowerCamelCase) > 0:
											words.append(bytearray(__lowerCamelCase).decode("""utf-8"""							,							errors="""replace"""))
								UpperCAmelCase__     :    Optional[Any]         =   "".join(__lowerCamelCase)
								return text



import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
    from sagemaker import Session, TrainingJobAnalytics
    from sagemaker.huggingface import HuggingFace







@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"     ,    "False")) is not True     ,    reason="Skipping test because should only be run when releasing minor transformers version"     ,    )
@pytest.mark.usefixtures("sm_env")
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 650, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "pytorch",
            "script": "run_ddp.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.7, "eval_loss": 0.6},
        },
        {
            "framework": "tensorflow",
            "script": "run_tf_dist.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.p3.16xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.6, "eval_loss": 0.7},
        },
    ])
class      lowerCAmelCase__       (				unittest.TestCase):




     '''simple docstring'''


     def  _lowerCamelCase (						self)			->				str:
            if self.framework == "pytorch":
                   subprocess.run(
                       F"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split() ,						encoding="utf-8" ,						check=__lowerCamelCase ,						)
            assert hasattr(self ,						"env")


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Tuple:
            _A      :       Dict				    = F"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
            # distributed data settings
            _A      :       Optional[Any]				    = {"smdistributed": {"dataparallel": {"enabled": True}}} if self.script != "run_ddp.py" else None

            # creates estimator
            return HuggingFace(
                entry_point=self.script ,						source_dir=self.env.test_path ,						role=self.env.role ,						image_uri=self.env.image_uri ,						base_job_name=__lowerCamelCase ,						instance_count=__lowerCamelCase ,						instance_type=self.instance_type ,						debugger_hook_config=__lowerCamelCase ,						hyperparameters={**self.env.distributed_hyperparameters, "model_name_or_path": self.model_name_or_path} ,						metric_definitions=self.env.metric_definitions ,						distribution=__lowerCamelCase ,						py_version="py36" ,						)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Optional[Any]:
            TrainingJobAnalytics(__lowerCamelCase).export_csv(F"{self.env.test_path}/{job_name}_metrics.csv")


     @parameterized.expand([(2,)])
     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Any:
            # create estimator
            _A      :       Union[str, Any]				    = self.create_estimator(__lowerCamelCase)

            # run training
            estimator.fit()

            # result dataframe
            _A      :       Optional[Any]				    = TrainingJobAnalytics(estimator.latest_training_job.name).dataframe()

            # extract kpis
            _A      :       List[Any]				    = list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"])
            _A      :       Dict				    = list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"])
            # get train time from SageMaker job, this includes starting, preprocessing, stopping
            _A      :       Optional[Any]				    = (
                Session().describe_training_job(estimator.latest_training_job.name).get("TrainingTimeInSeconds" ,						9_9_9_9_9_9)
            )

            # assert kpis
            assert train_runtime <= self.results["train_runtime"]
            assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy)
            assert all(t <= self.results["eval_loss"] for t in eval_loss)

            # dump tests result into json file to share in PR
            with open(F"{estimator.latest_training_job.name}.json" ,						"w") as outfile:
                   json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss} ,						__lowerCamelCase)
