
"""simple docstring"""




from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.gptj.modeling_tf_gptj import (
					    TFGPTJForCausalLM,
					    TFGPTJForQuestionAnswering,
					    TFGPTJForSequenceClassification,
					    TFGPTJModel,
					    shape_list,
					)
class SCREAMING_SNAKE_CASE       :






	'''simple docstring'''







	def __init__(  self :int       ,    __magic_name__ :List[str]			)			->   Optional[int]:

			'''simple docstring'''






			a__   		=     parent
			a__   		=     13
			a__   		=     7
			a__   		=     True
			a__   		=     True
			a__   		=     True
			a__   		=     True
			a__   		=     True
			a__   		=     99
			a__   		=     32
			a__   		=     4
			a__   		=     2
			a__   		=     4
			a__   		=     37
			a__   		=     '''gelu'''
			a__   		=     0.1
			a__   		=     0.1
			a__   		=     512
			a__   		=     16
			a__   		=     2
			a__   		=     0.02
			a__   		=     3
			a__   		=     4
			a__   		=     None
			a__   		=     self.vocab_size - 1
			a__   		=     self.vocab_size - 1
			a__   		=     self.vocab_size - 1


	def    _UpperCamelCase							(  self :str			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     ids_tensor([self.batch_size, self.seq_length]       ,    self.vocab_size			)

			a__   		=     None
			if self.use_input_mask:
					a__   		=     random_attention_mask([self.batch_size, self.seq_length]			)

			a__   		=     None
			if self.use_token_type_ids:
					a__   		=     ids_tensor([self.batch_size, self.seq_length]       ,    self.type_vocab_size			)

			a__   		=     None
			if self.use_mc_token_ids:
					a__   		=     ids_tensor([self.batch_size, self.num_choices]       ,    self.seq_length			)

			a__   		=     None
			a__   		=     None
			a__   		=     None
			if self.use_labels:
					a__   		=     ids_tensor([self.batch_size]       ,    self.type_sequence_label_size			)
					a__   		=     ids_tensor([self.batch_size, self.seq_length]       ,    self.num_labels			)
					a__   		=     ids_tensor([self.batch_size]       ,    self.num_choices			)

			a__   		=     GPTJConfig(
			    vocab_size=self.vocab_size       ,    n_embd=self.hidden_size       ,    n_layer=self.num_hidden_layers       ,    n_head=self.num_attention_heads       ,    intermediate_size=self.intermediate_size       ,    hidden_act=self.hidden_act       ,    hidden_dropout_prob=self.hidden_dropout_prob       ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,    n_positions=self.max_position_embeddings       ,    type_vocab_size=self.type_vocab_size       ,    initializer_range=self.initializer_range       ,    bos_token_id=self.bos_token_id       ,    eos_token_id=self.eos_token_id       ,    pad_token_id=self.pad_token_id       ,    rotary_dim=self.rotary_dim       ,    return_dict=__magic_name__       ,    )

			a__   		=     ids_tensor([self.num_hidden_layers, self.num_attention_heads]       ,    2			)

			return (
			    config,
			    input_ids,
			    input_mask,
			    head_mask,
			    token_type_ids,
			    mc_token_ids,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			)


	def    _UpperCamelCase							(  self :List[Any]       ,    __magic_name__ :str       ,    __magic_name__ :Tuple       ,    __magic_name__ :Tuple       ,    __magic_name__ :List[str]       ,    __magic_name__ :Any       ,    *__magic_name__ :List[str]			)			->   Tuple:

			'''simple docstring'''






			a__   		=     TFGPTJModel(config=__magic_name__			)
			a__   		=     {
			    '''input_ids''': input_ids,
			    '''attention_mask''': input_mask,
			    '''token_type_ids''': token_type_ids,
			}
			a__   		=     model(__magic_name__			)

			a__   		=     [input_ids, None, input_mask]  # None is the input for 'past'
			a__   		=     model(__magic_name__			)

			a__   		=     model(__magic_name__			)

			self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)			)


	def    _UpperCamelCase							(  self :str       ,    __magic_name__ :List[Any]       ,    __magic_name__ :str       ,    __magic_name__ :Optional[int]       ,    __magic_name__ :Optional[int]       ,    __magic_name__ :str       ,    *__magic_name__ :int			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     TFGPTJModel(config=__magic_name__			)

			# first forward pass
			a__   		=     model(__magic_name__       ,    token_type_ids=__magic_name__       ,    use_cache=__magic_name__			)
			a__   		=     model(__magic_name__       ,    token_type_ids=__magic_name__			)
			a__   		=     model(__magic_name__       ,    token_type_ids=__magic_name__       ,    use_cache=__magic_name__			)

			self.parent.assertTrue(len(__magic_name__			) == len(__magic_name__			)			)
			self.parent.assertTrue(len(__magic_name__			) == len(__magic_name__			) + 1			)

			a__		,	a__   		=     outputs.to_tuple()

			# create hypothetical next token and extent to next_input_ids
			a__   		=     ids_tensor((self.batch_size, 1)       ,    config.vocab_size			)
			a__   		=     ids_tensor([self.batch_size, 1]       ,    self.type_vocab_size			)

			# append to next input_ids and token_type_ids
			a__   		=     tf.concat([input_ids, next_tokens]       ,    axis=-1			)
			a__   		=     tf.concat([token_type_ids, next_token_types]       ,    axis=-1			)

			a__   		=     model(__magic_name__       ,    token_type_ids=__magic_name__			)['''last_hidden_state''']
			a__   		=     model(__magic_name__       ,    token_type_ids=__magic_name__       ,    past_key_values=__magic_name__			)[
			    '''last_hidden_state'''
			]

			# select random slice
			a__   		=     int(ids_tensor((1,)       ,    shape_list(__magic_name__			)[-1]			)			)
			a__   		=     output_from_no_past[:, -1, random_slice_idx]
			a__   		=     output_from_past[:, 0, random_slice_idx]

			# test that outputs are equal for slice
			tf.debugging.assert_near(__magic_name__       ,    __magic_name__       ,    rtol=1e-6			)


	def    _UpperCamelCase							(  self :Any       ,    __magic_name__ :Optional[int]       ,    __magic_name__ :Tuple       ,    __magic_name__ :int       ,    __magic_name__ :Dict       ,    __magic_name__ :Any       ,    *__magic_name__ :Union[str, Any]			)			->   Any:

			'''simple docstring'''






			a__   		=     TFGPTJModel(config=__magic_name__			)

			# create attention mask
			a__   		=     self.seq_length // 2
			a__   		=     tf.ones((self.batch_size, half_seq_length)       ,    dtype=tf.intaa			)
			a__   		=     tf.zeros((self.batch_size, self.seq_length - half_seq_length)       ,    dtype=tf.intaa			)
			a__   		=     tf.concat([attn_mask_begin, attn_mask_end]       ,    axis=1			)

			# first forward pass
			a__		,	a__   		=     model(__magic_name__       ,    attention_mask=__magic_name__			).to_tuple()

			# create hypothetical next token and extent to next_input_ids
			a__   		=     ids_tensor((self.batch_size, 1)       ,    config.vocab_size			)

			# change a random masked slice from input_ids
			a__   		=     ids_tensor((1,)       ,    __magic_name__			).numpy() + 1
			a__   		=     ids_tensor((self.batch_size, self.seq_length)       ,    config.vocab_size			)
			a__   		=     tf.range(self.seq_length			) == (self.seq_length - random_seq_idx_to_change)
			a__   		=     tf.transpose(
			    tf.broadcast_to(tf.expand_dims(__magic_name__       ,    -1			)       ,    (self.seq_length, self.batch_size)			)			)
			a__   		=     tf.where(__magic_name__       ,    __magic_name__       ,    __magic_name__			)

			# append to next input_ids and attn_mask
			a__   		=     tf.concat([input_ids, next_tokens]       ,    axis=-1			)
			a__   		=     tf.concat([attn_mask, tf.ones((shape_list(__magic_name__			)[0], 1)       ,    dtype=tf.intaa			)]       ,    axis=1			)

			# get two different outputs
			a__   		=     model(__magic_name__       ,    attention_mask=__magic_name__			)['''last_hidden_state''']
			a__   		=     model(__magic_name__       ,    past_key_values=__magic_name__       ,    attention_mask=__magic_name__			)[
			    '''last_hidden_state'''
			]

			# select random slice
			a__   		=     int(ids_tensor((1,)       ,    shape_list(__magic_name__			)[-1]			)			)
			a__   		=     output_from_no_past[:, -1, random_slice_idx]
			a__   		=     output_from_past[:, 0, random_slice_idx]

			# test that outputs are equal for slice
			tf.debugging.assert_near(__magic_name__       ,    __magic_name__       ,    rtol=1e-12			)


	def    _UpperCamelCase							(  self :Dict       ,    __magic_name__ :Any       ,    __magic_name__ :Dict       ,    __magic_name__ :List[str]       ,    __magic_name__ :List[str]       ,    __magic_name__ :Tuple       ,    *__magic_name__ :Optional[int]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     TFGPTJModel(config=__magic_name__			)

			a__   		=     input_ids[:1, :]
			a__   		=     input_mask[:1, :]
			a__   		=     token_type_ids[:1, :]
			a__   		=     1

			# first forward pass
			a__   		=     model(__magic_name__       ,    attention_mask=__magic_name__       ,    token_type_ids=__magic_name__       ,    use_cache=__magic_name__			)

			a__		,	a__   		=     outputs.to_tuple()

			# create hypothetical next token and extent to next_input_ids
			a__   		=     ids_tensor((self.batch_size, 3)       ,    config.vocab_size			)
			a__   		=     ids_tensor((self.batch_size, 3)       ,    2			)
			a__   		=     ids_tensor((self.batch_size, 3)       ,    self.type_vocab_size			)

			# append to next input_ids and token_type_ids
			a__   		=     tf.concat([input_ids, next_tokens]       ,    axis=-1			)
			a__   		=     tf.concat([input_mask, next_attn_mask]       ,    axis=-1			)
			a__   		=     tf.concat([token_type_ids, next_token_types]       ,    axis=-1			)

			a__   		=     model(
			    __magic_name__       ,    token_type_ids=__magic_name__       ,    attention_mask=__magic_name__			)['''last_hidden_state''']
			a__   		=     model(
			    __magic_name__       ,    token_type_ids=__magic_name__       ,    attention_mask=__magic_name__       ,    past_key_values=__magic_name__       ,    )['''last_hidden_state''']
			self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]			)

			# select random slice
			a__   		=     int(ids_tensor((1,)       ,    shape_list(__magic_name__			)[-1]			)			)
			a__   		=     output_from_no_past[:, -3:, random_slice_idx]
			a__   		=     output_from_past[:, :, random_slice_idx]

			# test that outputs are equal for slice
			tf.debugging.assert_near(__magic_name__       ,    __magic_name__       ,    rtol=1e-3			)


	def    _UpperCamelCase							(  self :Optional[int]       ,    __magic_name__ :Tuple       ,    __magic_name__ :Optional[Any]       ,    __magic_name__ :Tuple       ,    __magic_name__ :int       ,    __magic_name__ :Dict       ,    *__magic_name__ :Optional[Any]			)			->   List[str]:

			'''simple docstring'''






			a__   		=     TFGPTJForCausalLM(config=__magic_name__			)
			a__   		=     {
			    '''input_ids''': input_ids,
			    '''attention_mask''': input_mask,
			    '''token_type_ids''': token_type_ids,
			}
			a__   		=     model(__magic_name__			)
			self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)			)






	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     self.prepare_config_and_inputs()

			(
			    (
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,	(
			    a__			
			)		,				
			)   		=     config_and_inputs

			a__   		=     {
			    '''input_ids''': input_ids,
			    '''token_type_ids''': token_type_ids,
			    '''attention_mask''': input_mask,
			}
			return config, inputs_dict

@require_tf
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					,   lowerCAmelCase					,   lowerCAmelCase					,   unittest.TestCase					):






	'''simple docstring'''







	snake_case__							:		str         =		(
	    (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
	    if is_tf_available()
	    else ()
	)

	snake_case__							:		List[Any]         =		(TFGPTJForCausalLM,) if is_tf_available() else ()
	snake_case__							:		Any         =		(
	    {
	        'feature-extraction': TFGPTJModel,
	        'question-answering': TFGPTJForQuestionAnswering,
	        'text-classification': TFGPTJForSequenceClassification,
	        'text-generation': TFGPTJForCausalLM,
	        'zero-shot': TFGPTJForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	snake_case__							:		Tuple         =		False
	snake_case__							:		str         =		False
	snake_case__							:		List[Any]         =		False
	snake_case__							:		Union[str, Any]         =		False


	def    _UpperCamelCase							(  self :Union[str, Any]       ,    __magic_name__ :Tuple       ,    __magic_name__ :Tuple       ,    __magic_name__ :List[str]       ,    __magic_name__ :str       ,    __magic_name__ :Union[str, Any]			)			->   Optional[int]:

			'''simple docstring'''






			if (
			    pipeline_test_casse_name == "QAPipelineTests"
			    and tokenizer_name is not None
			    and not tokenizer_name.endswith('''Fast'''			)
			):
					# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
					# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
					# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
					return True

			return False


	def    _UpperCamelCase							(  self :Tuple			)			->   List[str]:

			'''simple docstring'''






			a__   		=     TFGPTJModelTester(self			)
			a__   		=     ConfigTester(self       ,    config_class=__magic_name__       ,    n_embd=37			)


	def    _UpperCamelCase							(  self :str			)			->   Any:

			'''simple docstring'''






			self.config_tester.run_common_tests()


	def    _UpperCamelCase							(  self :str			)			->   Tuple:

			'''simple docstring'''






			a__   		=     self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_gptj_model(*__magic_name__			)


	def    _UpperCamelCase							(  self :Any			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_gptj_model_past(*__magic_name__			)


	def    _UpperCamelCase							(  self :str			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_gptj_model_attention_mask_past(*__magic_name__			)


	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   Any:

			'''simple docstring'''






			a__   		=     self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_gptj_model_past_large_inputs(*__magic_name__			)


	def    _UpperCamelCase							(  self :List[Any]			)			->   Any:

			'''simple docstring'''






			a__   		=     self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_gptj_lm_head_model(*__magic_name__			)


	@slow
	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''			)			) > 0       ,    '''skip testing on GPU for now to avoid GPU OOM.'''       ,    )
	def    _UpperCamelCase							(  self :Any			)			->   Any:

			'''simple docstring'''






			a__   		=     TFGPTJModel.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    from_pt=__magic_name__			)
			self.assertIsNotNone(__magic_name__			)






	@unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''			)
	def    _UpperCamelCase							(  self :List[str]			)			->   Dict:

			'''simple docstring'''






			super().test_resize_token_embeddings()

@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class SCREAMING_SNAKE_CASE       (      unittest.TestCase					):






	'''simple docstring'''







	def    _UpperCamelCase							(  self :List[Any]			)			->   Any:

			'''simple docstring'''






			a__   		=     TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    from_pt=__magic_name__			)
			a__   		=     tf.convert_to_tensor([[464, 3290]]       ,    dtype=tf.intaa			)  # The dog
			# fmt: off
			# The dog is a man's best friend. It is a loyal companion, and it is a friend
			a__   		=     [464, 3290, 318, 257, 582, 338, 1266, 1545, 13, 632, 318, 257, 9112, 15185, 11, 290, 340, 318, 257, 1545]
			# fmt: on
			a__   		=     model.generate(__magic_name__       ,    do_sample=__magic_name__			)
			self.assertListEqual(output_ids[0].numpy().tolist()       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Tuple			)			->   List[str]:

			'''simple docstring'''






			a__   		=     AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    revision='''float16'''			)
			a__   		=     TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    revision='''float16'''       ,    from_pt=__magic_name__			)

			a__   		=     tokenizer('''Today is a nice day and'''       ,    return_tensors='''tf'''			)
			# forces the generation to happen on CPU, to avoid GPU-related quirks
			with tf.device(''':/CPU:0'''			):
					a__   		=     model.generate(**__magic_name__       ,    do_sample=__magic_name__       ,    seed=[42, 0]			)
			a__   		=     tokenizer.decode(output_ids[0]       ,    skip_special_tokens=__magic_name__			)

			a__   		=     '''Today is a nice day and I’m going to go for a walk. I’'''
			self.assertEqual(__magic_name__       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :List[str]			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    revision='''float16'''       ,    from_pt=__magic_name__			)
			a__   		=     AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B'''       ,    revision='''float16'''			)

			a__   		=     '''left'''

			# Define PAD Token = EOS Token = 50256
			a__   		=     tokenizer.eos_token
			a__   		=     model.config.eos_token_id

			# use different length sentences to test batching
			a__   		=     [
			    '''Hello, my dog is a little''',
			    '''Today, I''',
			]
			a__   		=     [
			    '''Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia''',
			    '''Today, I’m going to be talking about a topic that’''',
			]
			return model, tokenizer, sentences, expected_output_sentences


	def    _UpperCamelCase							(  self :Tuple			)			->   Tuple:

			'''simple docstring'''






			a__		,	a__		,	a__		,	a__   		=     self._get_beam_search_test_objects()

			a__   		=     tokenizer(__magic_name__       ,    return_tensors='''tf'''       ,    padding=__magic_name__			)
			a__   		=     model.generate(**__magic_name__       ,    do_sample=__magic_name__       ,    num_beams=2			)
			a__   		=     tokenizer.batch_decode(__magic_name__       ,    skip_special_tokens=__magic_name__			)
			self.assertListEqual(__magic_name__       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Optional[int]			)			->   List[Any]:

			'''simple docstring'''






			a__		,	a__		,	a__		,	a__   		=     self._get_beam_search_test_objects()

			a__   		=     tokenizer(__magic_name__       ,    return_tensors='''tf'''       ,    padding=__magic_name__			)
			a__   		=     tokenizer(sentences[0]       ,    return_tensors='''tf'''			)
			a__   		=     model.generate(**__magic_name__       ,    do_sample=__magic_name__       ,    num_beams=2			)
			a__   		=     (
			    shape_list(inputs_non_padded['''input_ids''']			)[-1]
			    - tf.reduce_sum(tf.cast(inputs['''attention_mask'''][-1]       ,    tf.intaa			)			).numpy()
			)
			a__   		=     tokenizer(sentences[1]       ,    return_tensors='''tf'''			)
			a__   		=     model.generate(
			    **__magic_name__       ,    do_sample=__magic_name__       ,    num_beams=2       ,    max_length=model.config.max_length - num_paddings			)
			a__   		=     tokenizer.decode(output_non_padded[0]       ,    skip_special_tokens=__magic_name__			)
			a__   		=     tokenizer.decode(output_padded[0]       ,    skip_special_tokens=__magic_name__			)
			self.assertListEqual(__magic_name__       ,    [non_padded_sentence, padded_sentence]			)






	def    _UpperCamelCase							(  self :Optional[int]			)			->   Tuple:

			'''simple docstring'''






			a__		,	a__		,	a__		,	a__   		=     self._get_beam_search_test_objects()

			a__   		=     tokenizer(__magic_name__       ,    return_tensors='''tf'''       ,    padding=__magic_name__			)
			a__   		=     tf.function(model.generate       ,    jit_compile=__magic_name__			)
			a__   		=     xla_generate(**__magic_name__       ,    do_sample=__magic_name__       ,    num_beams=2			)
			a__   		=     tokenizer.batch_decode(__magic_name__       ,    skip_special_tokens=__magic_name__			)
			self.assertListEqual(__magic_name__       ,    __magic_name__			)










import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class       _UpperCamelCase   (					tf.keras.layers.Layer		):
      def __init__(      self :int			,   lowerCamelCase :List			,   lowerCamelCase :bool			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :str = "longest"			,   lowerCamelCase :bool = True			,   lowerCamelCase :int = 512			,   lowerCamelCase :int = None			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   **lowerCamelCase :Optional[Any]			,   )  ->      List[str]:
       super().__init__()
       if use_fast_bert_tokenizer:
        UpperCAmelCase__							=    FastBertTokenizer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case_nfd_strip_accents=lowerCamelCase			,   **lowerCamelCase   )
       else:
        UpperCAmelCase__							=    tf.lookup.StaticVocabularyTable(
            tf.lookup.KeyValueTensorInitializer(
                keys=lowerCamelCase			,   key_dtype=tf.string			,   values=tf.range(tf.size(lowerCamelCase			,   out_type=tf.intaa   )			,   dtype=tf.intaa   )			,   value_dtype=tf.intaa			,   )			,   num_oov_buckets=1			,   )
        UpperCAmelCase__							=    BertTokenizerLayer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case=lowerCamelCase			,   **lowerCamelCase   )

       UpperCAmelCase__							=    vocab_list
       UpperCAmelCase__							=    do_lower_case
       UpperCAmelCase__							=    cls_token_id or vocab_list.index("[CLS]"   )
       UpperCAmelCase__							=    sep_token_id or vocab_list.index("[SEP]"   )
       UpperCAmelCase__							=    pad_token_id or vocab_list.index("[PAD]"   )
       UpperCAmelCase__							=    ShrinkLongestTrimmer(max_length - 3			,   axis=1   )  # Allow room for special tokens
       UpperCAmelCase__							=    max_length
       UpperCAmelCase__							=    padding
       UpperCAmelCase__							=    truncation
       UpperCAmelCase__							=    pad_to_multiple_of
       UpperCAmelCase__							=    return_token_type_ids
       UpperCAmelCase__							=    return_attention_mask
      @classmethod
      def 						UpperCAmelCase_    (      cls :List[str]			,   lowerCamelCase :"PreTrainedTokenizerBase"			,   **lowerCamelCase :Dict   )  ->      Dict:  # noqa: F821
       UpperCAmelCase__							=    kwargs.pop("do_lower_case"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
       UpperCAmelCase__							=    kwargs.pop("cls_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
       UpperCAmelCase__							=    kwargs.pop("sep_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
       UpperCAmelCase__							=    kwargs.pop("pad_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

       UpperCAmelCase__							=    tokenizer.get_vocab()
       UpperCAmelCase__							=    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,   key=lambda lowerCamelCase   : x[1]   )
       UpperCAmelCase__							=    [entry[0] for entry in vocab]
       return cls(
           vocab_list=lowerCamelCase			,   do_lower_case=lowerCamelCase			,   cls_token_id=lowerCamelCase			,   sep_token_id=lowerCamelCase			,   pad_token_id=lowerCamelCase			,   **lowerCamelCase			,   )
      @classmethod
      def 						UpperCAmelCase_    (      cls :Any			,   lowerCamelCase :Union[str, os.PathLike]			,   *lowerCamelCase :str			,   **lowerCamelCase :List[str]   )  ->      Tuple:
       try:
        UpperCAmelCase__							=    BertTokenizer.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       except:  # noqa: E722
        from .tokenization_bert_fast import BertTokenizerFast

        UpperCAmelCase__							=    BertTokenizerFast.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       return cls.from_tokenizer(lowerCamelCase			,   **lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :Optional[int]   )  ->      List[str]:
       if self.do_lower_case:
        UpperCAmelCase__							=    case_fold_utfa(lowerCamelCase   )
       UpperCAmelCase__							=    self.tf_tokenizer.tokenize(lowerCamelCase   )
       return tokens.merge_dims(1			,   -1   )
      def 						UpperCAmelCase_    (      self :Optional[Any]			,   lowerCamelCase :Tuple			,   lowerCamelCase :str=None			,   lowerCamelCase :Tuple=None			,   lowerCamelCase :Any=None			,   lowerCamelCase :Optional[int]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Tuple=None			,   )  ->      str:
       if padding is None:
        UpperCAmelCase__							=    self.padding
       if padding not in ("longest", "max_length"):
        raise ValueError("Padding must be either 'longest' or 'max_length'!"   )
       if max_length is not None and text_pair is not None:
        # Because we have to instantiate a Trimmer to do it properly
        raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"   )
       if max_length is None:
        UpperCAmelCase__							=    self.max_length
       if truncation is None:
        UpperCAmelCase__							=    self.truncation
       if pad_to_multiple_of is None:
        UpperCAmelCase__							=    self.pad_to_multiple_of
       if return_token_type_ids is None:
        UpperCAmelCase__							=    self.return_token_type_ids
       if return_attention_mask is None:
        UpperCAmelCase__							=    self.return_attention_mask
       if not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None and not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None:
        if text.shape.rank > 1:
         raise ValueError("text argument should not be multidimensional when a text pair is supplied!"   )
        if text_pair.shape.rank > 1:
         raise ValueError("text_pair should not be multidimensional!"   )
       if text.shape.rank == 2:
        UpperCAmelCase__			,			UpperCAmelCase__							=    text[:, 0], text[:, 1]
       UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
       if text_pair is None:  # Unpaired text
        if truncation:
         UpperCAmelCase__							=    text[:, : max_length - 2]  # Allow room for special tokens
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text,)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       else:  # Paired text
        UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
        if truncation:
         UpperCAmelCase__			,			UpperCAmelCase__							=    self.paired_trimmer.trim([text, text_pair]   )
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text, text_pair)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       if padding == "longest":
        UpperCAmelCase__							=    input_ids.bounding_shape(axis=1   )
        if pad_to_multiple_of is not None:
         # No ceiling division in tensorflow, so we negate floordiv instead
         UpperCAmelCase__							=    pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,   lowerCamelCase   ))
       else:
        UpperCAmelCase__							=    max_length

       UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
       UpperCAmelCase__							=    {"input_ids": input_ids}
       if return_attention_mask:
        UpperCAmelCase__							=    attention_mask
       if return_token_type_ids:
        UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(
            lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
        UpperCAmelCase__							=    token_type_ids
       return output







      def 						UpperCAmelCase_    (      self :Tuple   )  ->      Tuple:
       return {
           "vocab_list": self.vocab_list,
           "do_lower_case": self.do_lower_case,
           "cls_token_id": self.cls_token_id,
           "sep_token_id": self.sep_token_id,
           "pad_token_id": self.pad_token_id,
       }





