
'''simple docstring'''




from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          AutoTokenizer,
          TFAutoModelForCausalLM,
          TFAutoModelForSeqaSeqLM,
          TFAutoModelForSpeechSeqaSeq,
          TFAutoModelForVisionaSeq,
          TFBartForConditionalGeneration,
          TFLogitsProcessorList,
          TFMinLengthLogitsProcessor,
          tf_top_k_top_p_filtering,
      )

if is_tensorflow_text_available():
      import tensorflow_text as text





@require_tf
class  A				(							unittest.TestCase						):





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   List[Any]					=						tf.convert_to_tensor(
             [
                 [
                     8.2_220_991,  # 3rd highest value; idx. 0
                     -0.5_620_044,
                     5.23_229_752,
                     4.0_386_393,
                     -6.8_798_378,
                     -0.54_785_802,
                     -3.2_012_153,
                     2.92_777_176,
                     1.88_171_953,
                     7.35_341_276,  # 5th highest value; idx. 9
                     8.43_207_833,  # 2nd highest value; idx. 10
                     -9.85_711_836,
                     -5.96_209_236,
                     -1.13_039_161,
                     -7.1_115_294,
                     -0.8_369_633,
                     -5.3_186_408,
                     7.06_427_407,
                     0.81_369_344,
                     -0.82_023_817,
                     -5.9_179_796,
                     0.58_813_443,
                     -6.99_778_438,
                     4.71_551_189,
                     -0.18_771_637,
                     7.44_020_759,  # 4th highest value; idx. 25
                     9.38_450_987,  # 1st highest value; idx. 26
                     2.12_662_941,
                     -9.32_562_038,
                     2.35_652_522,
                 ],  # cummulative prob of 5 highest values <= 0.6
                 [
                     0.58_425_518,
                     4.53_139_238,
                     -5.57_510_464,
                     -6.28_030_699,
                     -7.19_529_503,
                     -4.02_122_551,
                     1.39_337_037,
                     -6.06_707_057,
                     1.59_480_517,
                     -9.643_119,
                     0.03_907_799,
                     0.67_231_762,
                     -8.88_206_726,
                     6.27_115_922,  # 4th highest value; idx. 13
                     2.28_520_723,
                     4.82_767_506,
                     4.30_421_368,
                     8.8_275_313,  # 2nd highest value; idx. 17
                     5.44_029_958,  # 5th highest value; idx. 18
                     -4.4_735_794,
                     7.38_579_536,  # 3rd highest value; idx. 20
                     -2.91_051_663,
                     2.61_946_077,
                     -2.5_674_762,
                     -9.48_959_302,
                     -4.02_922_645,
                     -1.35_416_918,
                     9.67_702_323,  # 1st highest value; idx. 27
                     -5.89_478_553,
                     1.85_370_467,
                 ],  # cummulative prob of 5 highest values <= 0.6
             ]						,			dtype=tf.floataa						,			)

         A       :   Union[str, Any]					=						tf.convert_to_tensor(
             [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]						,			dtype=tf.intaa						,			)  # expected non filtered idx as noted above

         A       :   str					=						tf.convert_to_tensor(
             [8.222_099, 7.3_534_126, 8.432_078, 7.4_402_075, 9.38_451, 6.271_159, 8.827_531, 5.4_402_995, 7.3_857_956, 9.677_023]						,			dtype=tf.floataa						,			)  # expected non filtered values as noted above

         A       :   str					=						tf_top_k_top_p_filtering(SCREAMING_SNAKE_CASE						,			top_k=10						,			top_p=0.6						,			min_tokens_to_keep=4	)

         A       :   Optional[int]					=						output[output != -float('''inf'''	)]
         A       :   Union[str, Any]					=						tf.cast(
             tf.where(tf.not_equal(SCREAMING_SNAKE_CASE						,			tf.constant(-float('''inf'''	)						,			dtype=tf.floataa	)	)	)						,			dtype=tf.intaa						,			)

         tf.debugging.assert_near(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			rtol=1e-12	)
         tf.debugging.assert_equal(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





@require_tf
class  A				(							unittest.TestCase    ,			__snake_case						):
     # setting framework_dependent_parameters needs to be gated, just like its contents' imports
     if is_tf_available():
          __magic_name__						=		{
              '''AutoModelForCausalLM''': TFAutoModelForCausalLM,
              '''AutoModelForSpeechSeq2Seq''': TFAutoModelForSpeechSeqaSeq,
              '''AutoModelForSeq2SeqLM''': TFAutoModelForSeqaSeqLM,
              '''AutoModelForVision2Seq''': TFAutoModelForVisionaSeq,
              '''LogitsProcessorList''': TFLogitsProcessorList,
              '''MinLengthLogitsProcessor''': TFMinLengthLogitsProcessor,
              '''create_tensor_fn''': tf.convert_to_tensor,
              '''floats_tensor''': floats_tensor,
              '''return_tensors''': '''tf''',
          }





     @slow
     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         A       :   Any					=						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''	)
         A       :   str					=						2
         A       :   List[str]					=						2





         class  A				(							tf.Module						):





              def __init__(							self						,			SCREAMING_SNAKE_CASE	) ->				Any:






                  """simple docstring"""






                  super(SCREAMING_SNAKE_CASE						,			self	).__init__()
                  A       :   str					=						model





              @tf.function(
                  input_signature=(
                      tf.TensorSpec((None, input_length)						,			tf.intaa						,			name='''input_ids'''	),
                      tf.TensorSpec((None, input_length)						,			tf.intaa						,			name='''attention_mask'''	),
                  )						,			jit_compile=SCREAMING_SNAKE_CASE						,			)
              def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	) ->				str:






                  """simple docstring"""






                  A       :   Dict					=						self.model.generate(
                      input_ids=SCREAMING_SNAKE_CASE						,			attention_mask=SCREAMING_SNAKE_CASE						,			max_new_tokens=SCREAMING_SNAKE_CASE						,			return_dict_in_generate=SCREAMING_SNAKE_CASE						,			)
                  return {"sequences": outputs["sequences"]}





         A       :   Tuple					=						[[2, 0], [102, 103]]
         A       :   List[Any]					=						[[1, 0], [1, 1]]
         A       :   Optional[int]					=						DummyModel(model=SCREAMING_SNAKE_CASE	)
         with tempfile.TemporaryDirectory() as tmp_dir:
             tf.saved_model.save(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			signatures={'''serving_default''': dummy_model.serving}	)
             A       :   Tuple					=						tf.saved_model.load(SCREAMING_SNAKE_CASE	).signatures['''serving_default''']
             for batch_size in range(1						,			len(SCREAMING_SNAKE_CASE	) + 1	):
                 A       :   List[Any]					=						{
                     '''input_ids''': tf.constant(dummy_input_ids[:batch_size]	),
                     '''attention_mask''': tf.constant(dummy_attention_masks[:batch_size]	),
                 }
                 A       :   List[Any]					=						serving_func(**SCREAMING_SNAKE_CASE	)['''sequences''']
                 A       :   Optional[Any]					=						test_model.generate(**SCREAMING_SNAKE_CASE						,			max_new_tokens=SCREAMING_SNAKE_CASE	)
                 tf.debugging.assert_equal(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     @slow
     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         A       :   str					=						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''	)
         A       :   Optional[Any]					=						1
         A       :   List[Any]					=						2





         class  A				(							tf.Module						):





              def __init__(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






                  """simple docstring"""






                  super(SCREAMING_SNAKE_CASE						,			self	).__init__()
                  A       :   Optional[Any]					=						model





              @tf.function(
                  input_signature=(
                      tf.TensorSpec((batch_size, None)						,			tf.intaa						,			name='''input_ids'''	),
                      tf.TensorSpec((batch_size, None)						,			tf.intaa						,			name='''attention_mask'''	),
                  )						,			jit_compile=SCREAMING_SNAKE_CASE						,			)
              def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	) ->				Dict:






                  """simple docstring"""






                  A       :   Any					=						self.model.generate(
                      input_ids=SCREAMING_SNAKE_CASE						,			attention_mask=SCREAMING_SNAKE_CASE						,			max_new_tokens=SCREAMING_SNAKE_CASE						,			return_dict_in_generate=SCREAMING_SNAKE_CASE						,			)
                  return {"sequences": outputs["sequences"]}





         A       :   Optional[int]					=						[[2], [102, 103]]
         A       :   List[str]					=						[[1], [1, 1]]
         A       :   Optional[int]					=						DummyModel(model=SCREAMING_SNAKE_CASE	)
         with tempfile.TemporaryDirectory() as tmp_dir:
             tf.saved_model.save(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			signatures={'''serving_default''': dummy_model.serving}	)
             A       :   Tuple					=						tf.saved_model.load(SCREAMING_SNAKE_CASE	).signatures['''serving_default''']
             for input_row in range(len(SCREAMING_SNAKE_CASE	)	):
                 A       :   List[str]					=						{
                     '''input_ids''': tf.constant([dummy_input_ids[input_row]]	),
                     '''attention_mask''': tf.constant([dummy_attention_masks[input_row]]	),
                 }
                 A       :   Tuple					=						serving_func(**SCREAMING_SNAKE_CASE	)['''sequences''']
                 A       :   List[str]					=						test_model.generate(**SCREAMING_SNAKE_CASE						,			max_new_tokens=SCREAMING_SNAKE_CASE	)
                 tf.debugging.assert_equal(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     @slow
     @require_tensorflow_text
     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         with tempfile.TemporaryDirectory() as tmp_dir:
             # file needed to load the TF tokenizer
             hf_hub_download(repo_id='''google/flan-t5-small'''						,			filename='''spiece.model'''						,			local_dir=SCREAMING_SNAKE_CASE	)





             class  A				(							tf.keras.layers.Layer						):





                  def __init__(							self	) ->				Any:






                      """simple docstring"""






                      super().__init__()
                      A       :   List[str]					=						text.SentencepieceTokenizer(
                          model=tf.io.gfile.GFile(os.path.join(SCREAMING_SNAKE_CASE						,			'''spiece.model'''	)						,			'''rb'''	).read()	)
                      A       :   List[Any]					=						TFAutoModelForSeqaSeqLM.from_pretrained('''hf-internal-testing/tiny-random-t5'''	)





                  def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			*SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	) ->				Union[str, Any]:






                      """simple docstring"""






                      A       :   Any					=						self.tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
                      A,  A       :   List[Any]					=						text.pad_model_inputs(
                          SCREAMING_SNAKE_CASE						,			max_seq_length=64						,			pad_value=self.model.config.pad_token_id	)
                      A       :   List[Any]					=						self.model.generate(input_ids=SCREAMING_SNAKE_CASE						,			attention_mask=SCREAMING_SNAKE_CASE	)
                      return self.tokenizer.detokenize(SCREAMING_SNAKE_CASE	)





             A       :   Tuple					=						CompleteSentenceTransformer()
             A       :   int					=						tf.keras.layers.Input(shape=(1,)						,			dtype=tf.string						,			name='''inputs'''	)
             A       :   Optional[int]					=						complete_model(SCREAMING_SNAKE_CASE	)
             A       :   Tuple					=						tf.keras.Model(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
             keras_model.save(SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         A       :   List[str]					=						{
             '''do_sample''': True,
             '''num_beams''': 1,
             '''top_p''': 0.7,
             '''top_k''': 10,
             '''temperature''': 0.7,
         }
         A       :   List[Any]					=						14

         A       :   Union[str, Any]					=						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''	)
         A       :   Optional[Any]					=						'''Hello, my dog is cute and'''
         A       :   List[str]					=						tokenizer(SCREAMING_SNAKE_CASE						,			return_tensors='''tf'''	)
         A       :   Union[str, Any]					=						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''	)

         A       :   Optional[Any]					=						638
         # forces the generation to happen on CPU, to avoid GPU-related quirks
         with tf.device(''':/CPU:0'''	):
             tf.random.set_seed(0	)
             A       :   int					=						model.generate(**SCREAMING_SNAKE_CASE						,			eos_token_id=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)
         self.assertTrue(expectation == len(generated_tokens[0]	)	)

         A       :   str					=						[638, 198]
         with tf.device(''':/CPU:0'''	):
             tf.random.set_seed(0	)
             A       :   Any					=						model.generate(**SCREAMING_SNAKE_CASE						,			eos_token_id=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)
         self.assertTrue(expectation == len(generated_tokens[0]	)	)





     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         A       :   str					=						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bart'''	)
         A       :   Tuple					=						'''Hugging Face is a technology company based in New York and Paris.'''
         A       :   Any					=						bart_tokenizer(SCREAMING_SNAKE_CASE						,			return_tensors='''tf'''	).input_ids
         A       :   Tuple					=						TFBartForConditionalGeneration.from_pretrained('''hf-internal-testing/tiny-random-bart'''	)
         A       :   Union[str, Any]					=						bart_model.generate(SCREAMING_SNAKE_CASE	).numpy()





         class  A				(							__snake_case						):





              def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=None						,			**SCREAMING_SNAKE_CASE	) ->				Tuple:






                  """simple docstring"""






                  return super().call(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)





         A       :   Tuple					=						FakeBart.from_pretrained('''hf-internal-testing/tiny-random-bart'''	)
         A       :   int					=						bart_model.generate(SCREAMING_SNAKE_CASE						,			foo='''bar'''	).numpy()
         self.assertTrue(np.array_equal(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)	)





         class  A				(							bart_model.model.encoder.__class__						):





              def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	) ->				Optional[int]:






                  """simple docstring"""






                  return super().call(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)





         A       :   Optional[int]					=						FakeEncoder(bart_model.config						,			bart_model.model.shared	)
         A       :   Tuple					=						fake_encoder

         # Normal generation still works (the output will be different because the encoder weights are different)
         A       :   Any					=						bart_model.generate(SCREAMING_SNAKE_CASE	).numpy()
         with self.assertRaises(SCREAMING_SNAKE_CASE	):
             # FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
             bart_model.generate(SCREAMING_SNAKE_CASE						,			foo='''bar'''	)






'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_rembert import RemBertTokenizer
else:
      lowercase			:				Optional[Any]    		=			None

lowercase			:				Tuple    		=			logging.get_logger(__name__)
lowercase			:				Dict    		=			{'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

lowercase			:				Tuple    		=			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

lowercase			:				List[str]    		=			{
    'google/rembert': 2_56,
}

lowercase			:				Dict    		=			'▁'





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		RemBertTokenizer





     def __init__(							self						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=True						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<unk>"						,			SCREAMING_SNAKE_CASE="[SEP]"						,			SCREAMING_SNAKE_CASE="<pad>"						,			SCREAMING_SNAKE_CASE="[CLS]"						,			SCREAMING_SNAKE_CASE="[MASK]"						,			**SCREAMING_SNAKE_CASE						,			) ->				List[str]:






         """simple docstring"""






         A       :   Optional[Any]					=						AddedToken(SCREAMING_SNAKE_CASE						,			lstrip=SCREAMING_SNAKE_CASE						,			rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	) else mask_token

         super().__init__(
             SCREAMING_SNAKE_CASE						,			tokenizer_file=SCREAMING_SNAKE_CASE						,			do_lower_case=SCREAMING_SNAKE_CASE						,			remove_space=SCREAMING_SNAKE_CASE						,			keep_accents=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			sep_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			cls_token=SCREAMING_SNAKE_CASE						,			mask_token=SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   List[Any]					=						do_lower_case
         A       :   str					=						remove_space
         A       :   int					=						keep_accents
         A       :   Union[str, Any]					=						vocab_file
         A       :   List[Any]					=						False if not self.vocab_file else True





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   List[Any]					=						[self.sep_token_id]
         A       :   Tuple					=						[self.cls_token_id]
         if token_ids_a is None:
             return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None						,			SCREAMING_SNAKE_CASE = False	) ->				List[int]:






         """simple docstring"""






         if already_has_special_tokens:
             if token_ids_a is not None:
                 raise ValueError(
                     '''You should not supply a second sequence if the provided sequence of '''
                     '''ids is already formatted with special tokens for the model.'''	)
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
             return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE	)) + [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				List[int]:






         """simple docstring"""






         A       :   Tuple					=						[self.sep_token_id]
         A       :   Dict					=						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
         return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         if not os.path.isdir(SCREAMING_SNAKE_CASE	):
             logger.error('''Vocabulary path ({}) should be a directory'''.format(SCREAMING_SNAKE_CASE	)	)
             return
         A       :   Any					=						os.path.join(
             SCREAMING_SNAKE_CASE						,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	):
             copyfile(self.vocab_file						,			SCREAMING_SNAKE_CASE	)

         return (out_vocab_file,)




