






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class     _SCREAMING_SNAKE_CASE       (			_a		,   unittest.TestCase      ):
	snake_case__							:     Optional[Any]     	=  TransfoXLTokenizer
	snake_case__							:     List[Any]     	=  False
	snake_case__							:     Tuple     	=  False





	def 						_A (		self				:					str	):
		super().setUp()

		UpperCamelCase     :Dict          =	[
		    """<unk>""",
		    """[CLS]""",
		    """[SEP]""",
		    """want""",
		    """unwanted""",
		    """wa""",
		    """un""",
		    """running""",
		    """,""",
		    """low""",
		    """l""",
		]
		UpperCamelCase     :str          =	os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES["""vocab_file"""]	)
		with open(self.vocab_file ,  """w""" ,  encoding="""utf-8"""	) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)





	def 						_A (		self				:					List[str] ,  **__lowerCamelCase				:					Any	):
		UpperCamelCase     :Any          =	True
		return TransfoXLTokenizer.from_pretrained(self.tmpdirname ,  **__lowerCamelCase	)





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					int	):
		UpperCamelCase     :List[Any]          =	"""<unk> UNwanted , running"""
		UpperCamelCase     :int          =	"""<unk> unwanted, running"""
		return input_text, output_text





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :List[str]          =	TransfoXLTokenizer(vocab_file=self.vocab_file ,  lower_case=__lowerCamelCase	)

		UpperCamelCase     :Any          =	tokenizer.tokenize("""<unk> UNwanted , running"""	)
		self.assertListEqual(__lowerCamelCase ,  ["""<unk>""", """unwanted""", """,""", """running"""]	)

		self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase	) ,  [0, 4, 8, 7]	)





	def 						_A (		self				:					Optional[Any]	):
		UpperCamelCase     :List[Any]          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """	) ,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]	)





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :int          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """	) ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)





	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Any          =	TransfoXLTokenizer(lower_case=__lowerCamelCase	)
		UpperCamelCase     :Optional[int]          =	"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
		UpperCamelCase     :Optional[int]          =	[
		    """Hello""",
		    """(""",
		    """bracket""",
		    """)""",
		    """and""",
		    """side""",
		    """@-@""",
		    """scrolled""",
		    """[""",
		    """and""",
		    """]""",
		    """Henry""",
		    """'s""",
		    """$""",
		    """5""",
		    """@,@""",
		    """000""",
		    """with""",
		    """3""",
		    """@.@""",
		    """34""",
		    """m""",
		    """.""",
		    """What""",
		    """'s""",
		    """up""",
		    """!""",
		    """?""",
		]

		self.assertListEqual(tokenizer.tokenize(__lowerCamelCase	) ,  __lowerCamelCase	)

		self.assertEqual(tokenizer.convert_tokens_to_string(__lowerCamelCase	) ,  __lowerCamelCase	)





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :Any          =	self.get_tokenizer()
		UpperCamelCase     :List[str]          =	len(__lowerCamelCase	)

		tokenizer.add_tokens(["""new1""", """new2"""]	)
		tokenizer.move_added_token("""new1""" ,  1	)

		# Check that moved token is not copied (duplicate)
		self.assertEqual(len(__lowerCamelCase	) ,  original_len + 2	)
		# Check that token is moved to specified id
		self.assertEqual(tokenizer.encode("""new1"""	) ,  [1]	)
		self.assertEqual(tokenizer.decode([1]	) ,  """new1"""	)








import re
import string

import numpy as np

import datasets


UpperCAmelCase_ :						Dict      =      '''
Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.
'''

UpperCAmelCase_ :						Any      =      '''
Args:
    predictions: List of predicted texts.
    references: List of reference texts.
    regexes_to_ignore: List, defaults to None. Regex expressions of characters to
        ignore when calculating the exact matches. Note: these regexes are removed
        from the input data before the changes based on the options below (e.g. ignore_case,
        ignore_punctuation, ignore_numbers) are applied.
    ignore_case: Boolean, defaults to False. If true, turns everything
        to lowercase so that capitalization differences are ignored.
    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before
        comparing predictions and references.
    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before
        comparing predictions and references.
Returns:
    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.
Examples:
    >>> exact_match = datasets.load_metric("exact_match")
    >>> refs = ["the cat", "theater", "YELLING", "agent007"]
    >>> preds = ["cat?", "theater", "yelling", "agent"]
    >>> results = exact_match.compute(references=refs, predictions=preds)
    >>> print(round(results["exact_match"], 1))
    25.0

    >>> exact_match = datasets.load_metric("exact_match")
    >>> refs = ["the cat", "theater", "YELLING", "agent007"]
    >>> preds = ["cat?", "theater", "yelling", "agent"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)
    >>> print(round(results["exact_match"], 1))
    50.0


    >>> exact_match = datasets.load_metric("exact_match")
    >>> refs = ["the cat", "theater", "YELLING", "agent007"]
    >>> preds = ["cat?", "theater", "yelling", "agent"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)
    >>> print(round(results["exact_match"], 1))
    75.0

    >>> exact_match = datasets.load_metric("exact_match")
    >>> refs = ["the cat", "theater", "YELLING", "agent007"]
    >>> preds = ["cat?", "theater", "yelling", "agent"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)
    >>> print(round(results["exact_match"], 1))
    100.0

    >>> exact_match = datasets.load_metric("exact_match")
    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]
    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]
    >>> results = exact_match.compute(references=refs, predictions=preds)
    >>> print(round(results["exact_match"], 1))
    33.3

'''

UpperCAmelCase_ :						Tuple      =      '''
'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,   _KWARGS_DESCRIPTION      )
class     _SCREAMING_SNAKE_CASE       (			datasets.Metric      ):





	def 						_A (		self				:					Optional[int]	):
		return datasets.MetricInfo(
		    description=_DESCRIPTION ,  citation=_CITATION ,  inputs_description=_KWARGS_DESCRIPTION ,  features=datasets.Features(
		        {
		            """predictions""": datasets.Value("""string""" ,  id="""sequence"""	),
		            """references""": datasets.Value("""string""" ,  id="""sequence"""	),
		        }	) ,  reference_urls=[] ,  )





	def 						_A (		self				:					Optional[int] ,  __lowerCamelCase				:					Union[str, Any] ,  __lowerCamelCase				:					List[str] ,  __lowerCamelCase				:					List[str]=None ,  __lowerCamelCase				:					Optional[int]=False ,  __lowerCamelCase				:					int=False ,  __lowerCamelCase				:					Optional[int]=False ,  ):
		if regexes_to_ignore is not None:
			for s in regexes_to_ignore:
				UpperCamelCase     :str          =	np.array([re.sub(__lowerCamelCase ,  """""" ,  __lowerCamelCase	) for x in predictions]	)
				UpperCamelCase     :Tuple          =	np.array([re.sub(__lowerCamelCase ,  """""" ,  __lowerCamelCase	) for x in references]	)
		else:
			UpperCamelCase     :Any          =	np.asarray(__lowerCamelCase	)
			UpperCamelCase     :str          =	np.asarray(__lowerCamelCase	)

		if ignore_case:
			UpperCamelCase     :Tuple          =	np.char.lower(__lowerCamelCase	)
			UpperCamelCase     :Any          =	np.char.lower(__lowerCamelCase	)

		if ignore_punctuation:
			UpperCamelCase     :Optional[int]          =	string.punctuation.maketrans("""""" ,  """""" ,  string.punctuation	)
			UpperCamelCase     :Optional[Any]          =	np.char.translate(__lowerCamelCase ,  table=__lowerCamelCase	)
			UpperCamelCase     :List[str]          =	np.char.translate(__lowerCamelCase ,  table=__lowerCamelCase	)

		if ignore_numbers:
			UpperCamelCase     :Tuple          =	string.digits.maketrans("""""" ,  """""" ,  string.digits	)
			UpperCamelCase     :Dict          =	np.char.translate(__lowerCamelCase ,  table=__lowerCamelCase	)
			UpperCamelCase     :Tuple          =	np.char.translate(__lowerCamelCase ,  table=__lowerCamelCase	)

		UpperCamelCase     :int          =	predictions == references

		return {"exact_match": np.mean(__lowerCamelCase	) * 100}
