



import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class    _lowercase (a_   , unittest.TestCase   ):
				'''simple docstring'''







				lowercase__     =    HerbertTokenizer
				lowercase__     =    HerbertTokenizerFast
				lowercase__     =    True

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									super().setUp()

									# Use a simpler test file without japanese/chinese characters
									with open(F"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"""  ,			encoding="utf-8"     ) as f_data:
														UpperCamelCase_			= f_data.read().replace("\n\n"  ,			"\n"     ).strip()

									UpperCamelCase_			= [
									    "<s>",
									    "</s>",
									    "l",
									    "o",
									    "w",
									    "e",
									    "r",
									    "s",
									    "t",
									    "i",
									    "d",
									    "n",
									    "w</w>",
									    "r</w>",
									    "t</w>",
									    "lo",
									    "low",
									    "er</w>",
									    "low</w>",
									    "lowest</w>",
									    "newer</w>",
									    "wider</w>",
									    ",</w>",
									    "<unk>",
									]
									UpperCamelCase_			= dict(zip(snake_case__  ,			range(len(snake_case__     )     )     )     )
									UpperCamelCase_			= ["l o 123", "lo w 1456", "e r</w> 1789", ""]

									UpperCamelCase_			= os.path.join(self.tmpdirname  ,			VOCAB_FILES_NAMES["vocab_file"]     )
									UpperCamelCase_			= os.path.join(self.tmpdirname  ,			VOCAB_FILES_NAMES["merges_file"]     )
									with open(self.vocab_file  ,			"w"     ) as fp:
														fp.write(json.dumps(snake_case__     )     )
									with open(self.merges_file  ,			"w"     ) as fp:
														fp.write("\n".join(snake_case__     )     )

				def    _lowerCamelCase       (			self  ,			snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= "lower newer"
									UpperCamelCase_			= "lower newer"
									return input_text, output_text

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.tokenizer_class(vocab_file=self.vocab_file  ,			merges_file=self.merges_file     )

									UpperCamelCase_			= "lower"
									UpperCamelCase_			= ["low", "er</w>"]
									UpperCamelCase_			= tokenizer.tokenize(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= tokens + ["<unk>"]
									UpperCamelCase_			= [16, 17, 23]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__     )  ,			snake_case__     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									if not self.test_rust_tokenizer:
														return

									UpperCamelCase_			= self.get_tokenizer()
									UpperCamelCase_			= self.get_rust_tokenizer()

									UpperCamelCase_			= "lower,newer"

									UpperCamelCase_			= tokenizer.tokenize(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.tokenize(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= self.get_rust_tokenizer()
									UpperCamelCase_			= tokenizer.encode(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

				@slow
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"     )

									UpperCamelCase_			= tokenizer.encode("konstruowanie sekwencji"  ,			add_special_tokens=snake_case__     )
									UpperCamelCase_			= tokenizer.encode("konstruowanie wielu sekwencji"  ,			add_special_tokens=snake_case__     )

									UpperCamelCase_			= tokenizer.build_inputs_with_special_tokens(snake_case__     )
									UpperCamelCase_			= tokenizer.build_inputs_with_special_tokens(snake_case__  ,			snake_case__     )

									assert encoded_sentence == [0] + text + [2]
									assert encoded_pair == [0] + text + [2] + text_a + [2]





def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) + 32 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) + 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) + 491.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) - 32) * 5 / 9 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(((float(_lowerCAmelCase) - 32) * 5 / 9) + 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) + 459.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) - 273.15 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(((float(_lowerCAmelCase) - 273.15) * 9 / 5) + 32 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 9 / 5) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) - 491.67) * 5 / 9 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round(float(_lowerCAmelCase) - 459.67 ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 5 / 9) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 1.25 + 273.15) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 2.25 + 32) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 1.25) ,						_lowerCAmelCase)



def  _lowerCAmelCase     (_lowerCAmelCase ,						_lowerCAmelCase = 2):
					return round((float(_lowerCAmelCase) * 2.25 + 32 + 459.67) ,						_lowerCAmelCase)


if __name__ == "__main__":
	import doctest

	doctest.testmod()
