






from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers.models.xglm.modeling_tf_xglm import (
	    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TFXGLMForCausalLM,
	    TFXGLMModel,
	)

@require_tf
class     _SCREAMING_SNAKE_CASE       :
	snake_case__							:     List[str]     	=  XGLMConfig
	snake_case__							:     Optional[Any]     	=  {}
	snake_case__							:     Optional[int]     	=  """gelu"""





	def __init__(		self				:					Union[str, Any] ,  __lowerCamelCase				:					str ,  __lowerCamelCase				:					Optional[Any]=14 ,  __lowerCamelCase				:					Union[str, Any]=7 ,  __lowerCamelCase				:					List[Any]=True ,  __lowerCamelCase				:					Any=True ,  __lowerCamelCase				:					int=True ,  __lowerCamelCase				:					Any=99 ,  __lowerCamelCase				:					Union[str, Any]=32 ,  __lowerCamelCase				:					List[Any]=2 ,  __lowerCamelCase				:					Optional[Any]=4 ,  __lowerCamelCase				:					Optional[Any]=37 ,  __lowerCamelCase				:					int="gelu" ,  __lowerCamelCase				:					int=0.1 ,  __lowerCamelCase				:					Any=0.1 ,  __lowerCamelCase				:					List[str]=512 ,  __lowerCamelCase				:					List[Any]=0.02 ,  ):
		UpperCamelCase     :Tuple          =	parent
		UpperCamelCase     :Optional[Any]          =	batch_size
		UpperCamelCase     :List[Any]          =	seq_length
		UpperCamelCase     :Union[str, Any]          =	is_training
		UpperCamelCase     :Tuple          =	use_input_mask
		UpperCamelCase     :List[str]          =	use_labels
		UpperCamelCase     :Union[str, Any]          =	vocab_size
		UpperCamelCase     :Tuple          =	d_model
		UpperCamelCase     :Any          =	num_hidden_layers
		UpperCamelCase     :Union[str, Any]          =	num_attention_heads
		UpperCamelCase     :Optional[Any]          =	ffn_dim
		UpperCamelCase     :List[Any]          =	activation_function
		UpperCamelCase     :str          =	activation_dropout
		UpperCamelCase     :List[str]          =	attention_dropout
		UpperCamelCase     :List[str]          =	max_position_embeddings
		UpperCamelCase     :Dict          =	initializer_range
		UpperCamelCase     :List[Any]          =	None
		UpperCamelCase     :str          =	0
		UpperCamelCase     :Any          =	2
		UpperCamelCase     :Any          =	1





	def 						_A (		self				:					int	):
		return XGLMConfig.from_pretrained("""facebook/xglm-564M"""	)





	def 						_A (		self				:					List[str]	):
		UpperCamelCase     :Union[str, Any]          =	tf.clip_by_value(
		    ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size	) ,  clip_value_min=0 ,  clip_value_max=3	)

		UpperCamelCase     :List[str]          =	None
		if self.use_input_mask:
			UpperCamelCase     :List[Any]          =	random_attention_mask([self.batch_size, self.seq_length]	)

		UpperCamelCase     :str          =	self.get_config()

		UpperCamelCase     :List[Any]          =	floats_tensor([self.num_hidden_layers, self.num_attention_heads] ,  2	)

		return (
		    config,
		    input_ids,
		    input_mask,
		    head_mask,
		)





	def 						_A (		self				:					str	):
		return XGLMConfig(
		    vocab_size=self.vocab_size ,  d_model=self.hidden_size ,  num_layers=self.num_hidden_layers ,  attention_heads=self.num_attention_heads ,  ffn_dim=self.ffn_dim ,  activation_function=self.activation_function ,  activation_dropout=self.activation_dropout ,  attention_dropout=self.attention_dropout ,  max_position_embeddings=self.max_position_embeddings ,  initializer_range=self.initializer_range ,  use_cache=__lowerCamelCase ,  bos_token_id=self.bos_token_id ,  eos_token_id=self.eos_token_id ,  pad_token_id=self.pad_token_id ,  return_dict=__lowerCamelCase ,  )





	def 						_A (		self				:					Union[str, Any]	):
		UpperCamelCase     :Optional[Any]          =	self.prepare_config_and_inputs()

		(
		    (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      (
		    UpperCamelCase
		)     ,      
		)     :Optional[Any]          =	config_and_inputs

		UpperCamelCase     :str          =	{
		    """input_ids""": input_ids,
		    """head_mask""": head_mask,
		}

		return config, inputs_dict

@require_tf
class     _SCREAMING_SNAKE_CASE       (			_a		,   _a		,   unittest.TestCase      ):
	snake_case__							:     int     	=  (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
	snake_case__							:     Dict     	=  (TFXGLMForCausalLM,) if is_tf_available() else ()
	snake_case__							:     Optional[Any]     	=  (
	    {"""feature-extraction""": TFXGLMModel, """text-generation""": TFXGLMForCausalLM} if is_tf_available() else {}
	)
	snake_case__							:     List[str]     	=  False
	snake_case__							:     Optional[int]     	=  False
	snake_case__							:     List[str]     	=  False





	def 						_A (		self				:					List[Any]	):
		UpperCamelCase     :Optional[Any]          =	TFXGLMModelTester(self	)
		UpperCamelCase     :List[Any]          =	ConfigTester(self ,  config_class=__lowerCamelCase ,  n_embd=37	)





	def 						_A (		self				:					int	):
		self.config_tester.run_common_tests()





	@slow
	def 						_A (		self				:					int	):
		for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			UpperCamelCase     :str          =	TFXGLMModel.from_pretrained(__lowerCamelCase	)
			self.assertIsNotNone(__lowerCamelCase	)





	@unittest.skip(reason="""Currently, model embeddings are going to undergo a major refactor."""	)
	def 						_A (		self				:					Dict	):
		super().test_resize_token_embeddings()

@require_tf
class     _SCREAMING_SNAKE_CASE       (			unittest.TestCase      ):





	@slow
	def 						_A (		self				:					Optional[Any] ,  __lowerCamelCase				:					Optional[int]=True	):
		UpperCamelCase     :Dict          =	TFXGLMForCausalLM.from_pretrained("""facebook/xglm-564M"""	)
		UpperCamelCase     :str          =	tf.convert_to_tensor([[2, 268, 9_865]] ,  dtype=tf.intaa	)  # The dog
		# </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
		# fmt: off
		UpperCamelCase     :int          =	[2, 268, 9_865, 67, 11, 1_988, 57_252, 9_865, 5, 984, 67, 1_988, 213_838, 1_658, 53, 70_446, 33, 6_657, 278, 1_581]
		# fmt: on
		UpperCamelCase     :str          =	model.generate(__lowerCamelCase ,  do_sample=__lowerCamelCase ,  num_beams=1	)
		if verify_outputs:
			self.assertListEqual(output_ids[0].numpy().tolist() ,  __lowerCamelCase	)





	@slow
	def 						_A (		self				:					Tuple	):
		UpperCamelCase     :Tuple          =	XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""	)
		UpperCamelCase     :Tuple          =	TFXGLMForCausalLM.from_pretrained("""facebook/xglm-564M"""	)

		tf.random.set_seed(0	)
		UpperCamelCase     :Optional[int]          =	tokenizer("""Today is a nice day and""" ,  return_tensors="""tf"""	)
		UpperCamelCase     :str          =	tokenized.input_ids
		# forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
		with tf.device(""":/CPU:0"""	):
			UpperCamelCase     :Any          =	model.generate(__lowerCamelCase ,  do_sample=__lowerCamelCase ,  seed=[7, 0]	)
		UpperCamelCase     :str          =	tokenizer.decode(output_ids[0] ,  skip_special_tokens=__lowerCamelCase	)

		UpperCamelCase     :Union[str, Any]          =	(
		    """Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due"""
		)
		self.assertEqual(__lowerCamelCase ,  __lowerCamelCase	)





	@slow
	def 						_A (		self				:					int	):
		UpperCamelCase     :List[str]          =	TFXGLMForCausalLM.from_pretrained("""facebook/xglm-564M"""	)
		UpperCamelCase     :Tuple          =	XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""	)

		UpperCamelCase     :str          =	"""left"""

		# use different length sentences to test batching
		UpperCamelCase     :Union[str, Any]          =	[
		    """This is an extremelly long sentence that only exists to test the ability of the model to cope with """
		    """left-padding, such as in batched generation. The output for the sequence below should be the same """
		    """regardless of whether left padding is applied or not. When""",
		    """Hello, my dog is a little""",
		]

		UpperCamelCase     :str          =	tokenizer(__lowerCamelCase ,  return_tensors="""tf""" ,  padding=__lowerCamelCase	)
		UpperCamelCase     :Optional[int]          =	inputs["""input_ids"""]

		UpperCamelCase     :Dict          =	model.generate(input_ids=__lowerCamelCase ,  attention_mask=inputs["""attention_mask"""] ,  max_new_tokens=12	)

		UpperCamelCase     :List[str]          =	tokenizer(sentences[0] ,  return_tensors="""tf"""	).input_ids
		UpperCamelCase     :List[Any]          =	model.generate(input_ids=__lowerCamelCase ,  max_new_tokens=12	)

		UpperCamelCase     :str          =	tokenizer(sentences[1] ,  return_tensors="""tf"""	).input_ids
		UpperCamelCase     :str          =	model.generate(input_ids=__lowerCamelCase ,  max_new_tokens=12	)

		UpperCamelCase     :Any          =	tokenizer.batch_decode(__lowerCamelCase ,  skip_special_tokens=__lowerCamelCase	)
		UpperCamelCase     :Union[str, Any]          =	tokenizer.decode(output_non_padded[0] ,  skip_special_tokens=__lowerCamelCase	)
		UpperCamelCase     :List[Any]          =	tokenizer.decode(output_padded[0] ,  skip_special_tokens=__lowerCamelCase	)

		UpperCamelCase     :List[Any]          =	[
		    """This is an extremelly long sentence that only exists to test the ability of the model to cope with """
		    """left-padding, such as in batched generation. The output for the sequence below should be the same """
		    """regardless of whether left padding is applied or not. When left padding is applied, the sequence will be """
		    """a single""",
		    """Hello, my dog is a little bit of a shy one, but he is very friendly""",
		]
		self.assertListEqual(__lowerCamelCase ,  __lowerCamelCase	)
		self.assertListEqual(__lowerCamelCase ,  [non_padded_sentence, padded_sentence]	)








import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
	import bsa
	from bsa import BeautifulSoup


UpperCAmelCase_ :						Any      =      logging.get_logger(__name__)

class     _SCREAMING_SNAKE_CASE       (			_a      ):





	def __init__(		self				:					Optional[int] ,  **__lowerCamelCase				:					Optional[int]	):
		requires_backends(self ,  ["""bs4"""]	)
		super().__init__(**__lowerCamelCase	)





	def 						_A (		self				:					List[str] ,  __lowerCamelCase				:					Any	):
		UpperCamelCase     :Optional[int]          =	[]
		UpperCamelCase     :List[str]          =	[]
		UpperCamelCase     :Union[str, Any]          =	element if element.name else element.parent
		for parent in child.parents:  # type: bs4.element.Tag
			UpperCamelCase     :Optional[Any]          =	parent.find_all(child.name ,  recursive=__lowerCamelCase	)
			xpath_tags.append(child.name	)
			xpath_subscripts.append(
			    0 if 1 == len(__lowerCamelCase	) else next(i for i, s in enumerate(__lowerCamelCase ,  1	) if s is child	)	)
			UpperCamelCase     :Any          =	parent
		xpath_tags.reverse()
		xpath_subscripts.reverse()
		return xpath_tags, xpath_subscripts





	def 						_A (		self				:					Any ,  __lowerCamelCase				:					Tuple	):
		UpperCamelCase     :Any          =	BeautifulSoup(__lowerCamelCase ,  """html.parser"""	)

		UpperCamelCase     :Union[str, Any]          =	[]
		UpperCamelCase     :Tuple          =	[]
		UpperCamelCase     :Tuple          =	[]

		for element in html_code.descendants:
			if type(__lowerCamelCase	) == bsa.element.NavigableString:
				if type(element.parent	) != bsa.element.Tag:
					continue

				UpperCamelCase     :Any          =	html.unescape(__lowerCamelCase	).strip()
				if not text_in_this_tag:
					continue

				all_doc_strings.append(__lowerCamelCase	)

				UpperCamelCase     ,      UpperCamelCase     :Optional[Any]          =	self.xpath_soup(__lowerCamelCase	)
				stringaxtag_seq.append(__lowerCamelCase	)
				stringaxsubs_seq.append(__lowerCamelCase	)

		if len(__lowerCamelCase	) != len(__lowerCamelCase	):
			raise ValueError("""Number of doc strings and xtags does not correspond"""	)
		if len(__lowerCamelCase	) != len(__lowerCamelCase	):
			raise ValueError("""Number of doc strings and xsubs does not correspond"""	)

		return all_doc_strings, stringaxtag_seq, stringaxsubs_seq





	def 						_A (		self				:					int ,  __lowerCamelCase				:					List[Any] ,  __lowerCamelCase				:					List[str]	):
		UpperCamelCase     :Tuple          =	""""""
		for tagname, subs in zip(__lowerCamelCase ,  __lowerCamelCase	):
			xpath += F"""/{tagname}"""
			if subs != 0:
				xpath += F"""[{subs}]"""
		return xpath





	def __call__(		self				:					Any ,  __lowerCamelCase				:					Dict	):
		UpperCamelCase     :Any          =	False

		# Check that strings has a valid type
		if isinstance(__lowerCamelCase ,  __lowerCamelCase	):
			UpperCamelCase     :List[Any]          =	True
		elif isinstance(__lowerCamelCase ,  (list, tuple)	):
			if len(__lowerCamelCase	) == 0 or isinstance(html_strings[0] ,  __lowerCamelCase	):
				UpperCamelCase     :Any          =	True

		if not valid_strings:
			raise ValueError(
			    """HTML strings must of type `str`, `List[str]` (batch of examples), """
			    F"""but is of type {type(__lowerCamelCase	)}."""	)

		UpperCamelCase     :str          =	bool(isinstance(__lowerCamelCase ,  (list, tuple)	) and (isinstance(html_strings[0] ,  __lowerCamelCase	))	)

		if not is_batched:
			UpperCamelCase     :Any          =	[html_strings]

		# Get nodes + xpaths
		UpperCamelCase     :Union[str, Any]          =	[]
		UpperCamelCase     :str          =	[]
		for html_string in html_strings:
			UpperCamelCase     ,      UpperCamelCase     ,      UpperCamelCase     :int          =	self.get_three_from_single(__lowerCamelCase	)
			nodes.append(__lowerCamelCase	)
			UpperCamelCase     :int          =	[]
			for node, tag_list, sub_list in zip(__lowerCamelCase ,  __lowerCamelCase ,  __lowerCamelCase	):
				UpperCamelCase     :str          =	self.construct_xpath(__lowerCamelCase ,  __lowerCamelCase	)
				xpath_strings.append(__lowerCamelCase	)
			xpaths.append(__lowerCamelCase	)

		# return as Dict
		UpperCamelCase     :Optional[int]          =	{"""nodes""": nodes, """xpaths""": xpaths}
		UpperCamelCase     :Any          =	BatchFeature(data=__lowerCamelCase ,  tensor_type=__lowerCamelCase	)

		return encoded_inputs
