


"""simple docstring"""







import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


a__				:							Optional[int]						=     get_tests_dir('''fixtures/test_sentencepiece_bpe.model''')




class  UpperCamelCase_	( UpperCamelCase			,  unittest.TestCase):

	"""simple docstring"""





	snake_case__				:  Optional[int]    					=			BartphoTokenizer
	snake_case__				:  Union[str, Any]    					=			False
	snake_case__				:  Optional[int]    					=			True




	def   UpperCAmelCase_      ( self			:				List[str]   )       ->	int:
						super().setUp()

						__SCREAMING_SNAKE_CASE						   =				["▁This", "▁is", "▁a", "▁t", "est"]
						__SCREAMING_SNAKE_CASE						   =				dict(zip(UpperCAmelCase__			,			range(len(UpperCAmelCase__   )   )   )   )
						__SCREAMING_SNAKE_CASE						   =				{"unk_token": "<unk>"}

						__SCREAMING_SNAKE_CASE						   =				os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["monolingual_vocab_file"]   )
						with open(self.monolingual_vocab_file			,			"w"			,			encoding="utf-8"   ) as fp:
											for token in vocab_tokens:
																fp.write(F"""{token} {vocab_tokens[token]}\n"""   )

						__SCREAMING_SNAKE_CASE						   =				BartphoTokenizer(UpperCAmelCase__			,			self.monolingual_vocab_file			,			**self.special_tokens_map   )
						tokenizer.save_pretrained(self.tmpdirname   )




	def   UpperCAmelCase_      ( self			:				str			,			**UpperCAmelCase__			:				int   )       ->	int:
						kwargs.update(self.special_tokens_map   )
						return BartphoTokenizer.from_pretrained(self.tmpdirname			,			**UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				List[Any]			,			UpperCAmelCase__			:				Union[str, Any]   )       ->	List[Any]:
						__SCREAMING_SNAKE_CASE						   =				"This is a là test"
						__SCREAMING_SNAKE_CASE						   =				"This is a<unk><unk> test"
						return input_text, output_text




	def   UpperCAmelCase_      ( self			:				List[str]   )       ->	str:
						__SCREAMING_SNAKE_CASE						   =				BartphoTokenizer(UpperCAmelCase__			,			self.monolingual_vocab_file			,			**self.special_tokens_map   )
						__SCREAMING_SNAKE_CASE						   =				"This is a là test"
						__SCREAMING_SNAKE_CASE						   =				"▁This ▁is ▁a ▁l à ▁t est".split()
						__SCREAMING_SNAKE_CASE						   =				tokenizer.tokenize(UpperCAmelCase__   )
						self.assertListEqual(UpperCAmelCase__			,			UpperCAmelCase__   )

						__SCREAMING_SNAKE_CASE						   =				tokens + [tokenizer.unk_token]
						__SCREAMING_SNAKE_CASE						   =				[4, 5, 6, 3, 3, 7, 8, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__   )			,			UpperCAmelCase__   )




"""simple docstring"""







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
		import torch

		from transformers.generation import DisjunctiveConstraint




@require_torch
class  UpperCamelCase_	( unittest.TestCase):

	"""simple docstring"""





	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	List[str]:
						# For consistency across different places the DisjunctiveConstraint is called,
						# dc.token_ids is a list of integers. It is also initialized only by integers.

						__SCREAMING_SNAKE_CASE						   =				[[1, 2, 4], [1, 2, 3, 4]]
						__SCREAMING_SNAKE_CASE						   =				DisjunctiveConstraint(UpperCAmelCase__   )
						self.assertTrue(isinstance(dc.token_ids			,			UpperCAmelCase__   )   )

						with self.assertRaises(UpperCAmelCase__   ):
											DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]   )   )

						with self.assertRaises(UpperCAmelCase__   ):
											DisjunctiveConstraint([torch.LongTensor([1, 2, 4]   ), torch.LongTensor([1, 2, 3, 4, 5]   )]   )




	def   UpperCAmelCase_      ( self			:				Any   )       ->	int:
						# We can't have constraints that are complete subsets of another. This leads to a preverse
						# interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
						# It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
						# fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
						# will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
						__SCREAMING_SNAKE_CASE						   =				[[1, 2], [1, 2, 3, 4]]

						with self.assertRaises(UpperCAmelCase__   ):
											DisjunctiveConstraint(UpperCAmelCase__   )  # fails here




	def   UpperCAmelCase_      ( self			:				List[Any]   )       ->	Any:
						__SCREAMING_SNAKE_CASE						   =				[[1, 2, 3], [1, 2, 4]]

						__SCREAMING_SNAKE_CASE						   =				DisjunctiveConstraint(UpperCAmelCase__   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(1   )
						__SCREAMING_SNAKE_CASE						   =				stepped is True and completed is False and reset is False
						self.assertTrue(UpperCAmelCase__   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.current_seq == [1]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(2   )
						__SCREAMING_SNAKE_CASE						   =				stepped is True and completed is False and reset is False
						self.assertTrue(UpperCAmelCase__   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.current_seq == [1, 2]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(3   )
						__SCREAMING_SNAKE_CASE						   =				stepped is True and completed is True and reset is False
						self.assertTrue(UpperCAmelCase__   )
						self.assertTrue(dc.completed   )  # Completed!
						self.assertTrue(dc.current_seq == [1, 2, 3]   )




	def   UpperCAmelCase_      ( self			:				str   )       ->	List[str]:
						__SCREAMING_SNAKE_CASE						   =				[[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

						__SCREAMING_SNAKE_CASE						   =				DisjunctiveConstraint(UpperCAmelCase__   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(1   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.current_seq == [1]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(2   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.current_seq == [1, 2]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(4   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.current_seq == [1, 2, 4]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(5   )
						self.assertTrue(dc.completed   )  # Completed!
						self.assertTrue(dc.current_seq == [1, 2, 4, 5]   )

						dc.reset()

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(1   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.remaining() == 3   )
						self.assertTrue(dc.current_seq == [1]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(2   )
						self.assertTrue(not dc.completed   )
						self.assertTrue(dc.remaining() == 2   )
						self.assertTrue(dc.current_seq == [1, 2]   )

						__SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE   ,  __SCREAMING_SNAKE_CASE						   =				dc.update(5   )
						self.assertTrue(dc.completed   )  # Completed!
						self.assertTrue(dc.remaining() == 0   )
						self.assertTrue(dc.current_seq == [1, 2, 5]   )
