

from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      AutoTokenizer,
      TFAutoModelForCausalLM,
      TFAutoModelForSeqaSeqLM,
      TFAutoModelForSpeechSeqaSeq,
      TFAutoModelForVisionaSeq,
      TFBartForConditionalGeneration,
      TFLogitsProcessorList,
      TFMinLengthLogitsProcessor,
      tf_top_k_top_p_filtering,
  )

if is_tensorflow_text_available():
  import tensorflow_text as text





@require_tf
class  __lowerCAmelCase     (					unittest.TestCase							):




  def   A__     (				self					)  -> str:
        '''simple docstring'''






        _lowercase	       =tf.convert_to_tensor(
            [
                [
                    8.2220991,  # 3rd highest value; idx. 0
                    -0.5620044,
                    5.23229752,
                    4.0386393,
                    -6.8798378,
                    -0.54785802,
                    -3.2012153,
                    2.92777176,
                    1.88171953,
                    7.35341276,  # 5th highest value; idx. 9
                    8.43207833,  # 2nd highest value; idx. 10
                    -9.85711836,
                    -5.96209236,
                    -1.13039161,
                    -7.1115294,
                    -0.8369633,
                    -5.3186408,
                    7.06427407,
                    0.81369344,
                    -0.82023817,
                    -5.9179796,
                    0.58813443,
                    -6.99778438,
                    4.71551189,
                    -0.18771637,
                    7.44020759,  # 4th highest value; idx. 25
                    9.38450987,  # 1st highest value; idx. 26
                    2.12662941,
                    -9.32562038,
                    2.35652522,
                ],  # cummulative prob of 5 highest values <= 0.6
                [
                    0.58425518,
                    4.53139238,
                    -5.57510464,
                    -6.28030699,
                    -7.19529503,
                    -4.02122551,
                    1.39337037,
                    -6.06707057,
                    1.59480517,
                    -9.643119,
                    0.03907799,
                    0.67231762,
                    -8.88206726,
                    6.27115922,  # 4th highest value; idx. 13
                    2.28520723,
                    4.82767506,
                    4.30421368,
                    8.8275313,  # 2nd highest value; idx. 17
                    5.44029958,  # 5th highest value; idx. 18
                    -4.4735794,
                    7.38579536,  # 3rd highest value; idx. 20
                    -2.91051663,
                    2.61946077,
                    -2.5674762,
                    -9.48959302,
                    -4.02922645,
                    -1.35416918,
                    9.67702323,  # 1st highest value; idx. 27
                    -5.89478553,
                    1.85370467,
                ],  # cummulative prob of 5 highest values <= 0.6
            ]							,   dtype=tf.floataa							,   )

        _lowercase	       =tf.convert_to_tensor(
            [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]							,   dtype=tf.intaa							,   )  # expected non filtered idx as noted above

        _lowercase	       =tf.convert_to_tensor(
            [8.222099, 7.3534126, 8.432078, 7.4402075, 9.38451, 6.271159, 8.827531, 5.4402995, 7.3857956, 9.677023]							,   dtype=tf.floataa							,   )  # expected non filtered values as noted above

        _lowercase	       =tf_top_k_top_p_filtering(snake_case_							,   top_k=10							,   top_p=0.6							,   min_tokens_to_keep=4					)

        _lowercase	       =output[output != -float('inf'					)]
        _lowercase	       =tf.cast(
            tf.where(tf.not_equal(snake_case_							,   tf.constant(-float('inf'					)							,   dtype=tf.floataa					)					)					)							,   dtype=tf.intaa							,   )

        tf.debugging.assert_near(snake_case_							,   snake_case_							,   rtol=1e-12					)
        tf.debugging.assert_equal(snake_case_							,   snake_case_					)





@require_tf
class  __lowerCAmelCase     (					unittest.TestCase			,			_a							):







  if is_tf_available():
    _a									=  {
        """AutoModelForCausalLM""": TFAutoModelForCausalLM,
        """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
        """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
        """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
        """LogitsProcessorList""": TFLogitsProcessorList,
        """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
        """create_tensor_fn""": tf.convert_to_tensor,
        """floats_tensor""": floats_tensor,
        """return_tensors""": """tf""",
    }




  @slow
  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'					)
        _lowercase	       =2
        _lowercase	       =2





        class  __lowerCAmelCase     (					tf.Module							):




          def __init__(				self							,   lowerCAmelCase					)  -> int:
                '''simple docstring'''






                super(snake_case_							,   self					).__init__()
                _lowercase	       =model




          @tf.function(
              input_signature=(
                  tf.TensorSpec((None, input_length)							,   tf.intaa							,   name='input_ids'					),
                  tf.TensorSpec((None, input_length)							,   tf.intaa							,   name='attention_mask'					),
              )							,   jit_compile=snake_case_							,   )
          def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase					)  -> Any:
                '''simple docstring'''






                _lowercase	       =self.model.generate(
                    input_ids=snake_case_							,   attention_mask=snake_case_							,   max_new_tokens=snake_case_							,   return_dict_in_generate=snake_case_							,   )
                return {"sequences": outputs["sequences"]}
        _lowercase	       =[[2, 0], [102, 103]]
        _lowercase	       =[[1, 0], [1, 1]]
        _lowercase	       =DummyModel(model=snake_case_					)
        with tempfile.TemporaryDirectory() as tmp_dir:
              tf.saved_model.save(snake_case_							,   snake_case_							,   signatures={'serving_default': dummy_model.serving}					)
              _lowercase	       =tf.saved_model.load(snake_case_					).signatures["""serving_default"""]
              for batch_size in range(1							,   len(snake_case_					) + 1					):
                    _lowercase	       ={
                        """input_ids""": tf.constant(dummy_input_ids[:batch_size]					),
                        """attention_mask""": tf.constant(dummy_attention_masks[:batch_size]					),
                    }
                    _lowercase	       =serving_func(**snake_case_					)["""sequences"""]
                    _lowercase	       =test_model.generate(**snake_case_							,   max_new_tokens=snake_case_					)
                    tf.debugging.assert_equal(snake_case_							,   snake_case_					)




  @slow
  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'					)
        _lowercase	       =1
        _lowercase	       =2





        class  __lowerCAmelCase     (					tf.Module							):




          def __init__(				self							,   lowerCAmelCase					)  -> Dict:
                '''simple docstring'''






                super(snake_case_							,   self					).__init__()
                _lowercase	       =model




          @tf.function(
              input_signature=(
                  tf.TensorSpec((batch_size, None)							,   tf.intaa							,   name='input_ids'					),
                  tf.TensorSpec((batch_size, None)							,   tf.intaa							,   name='attention_mask'					),
              )							,   jit_compile=snake_case_							,   )
          def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase					)  -> List[Any]:
                '''simple docstring'''






                _lowercase	       =self.model.generate(
                    input_ids=snake_case_							,   attention_mask=snake_case_							,   max_new_tokens=snake_case_							,   return_dict_in_generate=snake_case_							,   )
                return {"sequences": outputs["sequences"]}
        _lowercase	       =[[2], [102, 103]]
        _lowercase	       =[[1], [1, 1]]
        _lowercase	       =DummyModel(model=snake_case_					)
        with tempfile.TemporaryDirectory() as tmp_dir:
              tf.saved_model.save(snake_case_							,   snake_case_							,   signatures={'serving_default': dummy_model.serving}					)
              _lowercase	       =tf.saved_model.load(snake_case_					).signatures["""serving_default"""]
              for input_row in range(len(snake_case_					)					):
                    _lowercase	       ={
                        """input_ids""": tf.constant([dummy_input_ids[input_row]]					),
                        """attention_mask""": tf.constant([dummy_attention_masks[input_row]]					),
                    }
                    _lowercase	       =serving_func(**snake_case_					)["""sequences"""]
                    _lowercase	       =test_model.generate(**snake_case_							,   max_new_tokens=snake_case_					)
                    tf.debugging.assert_equal(snake_case_							,   snake_case_					)




  @slow
  @require_tensorflow_text
  def   A__     (				self					)  -> List[Any]:
        '''simple docstring'''






        with tempfile.TemporaryDirectory() as tmp_dir:
              # file needed to load the TF tokenizer
              hf_hub_download(repo_id='google/flan-t5-small'							,   filename='spiece.model'							,   local_dir=snake_case_					)





              class  __lowerCAmelCase     (					tf.keras.layers.Layer							):




                def __init__(				self					)  -> Any:
                      '''simple docstring'''






                      super().__init__()
                      _lowercase	       =text.SentencepieceTokenizer(
                          model=tf.io.gfile.GFile(os.path.join(snake_case_							,   'spiece.model'					)							,   'rb'					).read()					)
                      _lowercase	       =TFAutoModelForSeqaSeqLM.from_pretrained('hf-internal-testing/tiny-random-t5'					)




                def   A__     (				self							,   lowerCAmelCase							,   *lowerCAmelCase							,   **lowerCAmelCase					)  -> Union[str, Any]:
                      '''simple docstring'''






                      _lowercase	       =self.tokenizer.tokenize(snake_case_					)
                      _lowercase	       =text.pad_model_inputs(
                          snake_case_							,   max_seq_length=64							,   pad_value=self.model.config.pad_token_id					)
                      _lowercase	       =self.model.generate(input_ids=snake_case_							,   attention_mask=snake_case_					)
                      return self.tokenizer.detokenize(snake_case_					)
              _lowercase	       =CompleteSentenceTransformer()
              _lowercase	       =tf.keras.layers.Input(shape=(1,)							,   dtype=tf.string							,   name='inputs'					)
              _lowercase	       =complete_model(snake_case_					)
              _lowercase	       =tf.keras.Model(snake_case_							,   snake_case_					)
              keras_model.save(snake_case_					)




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        _lowercase	       ={
            """do_sample""": True,
            """num_beams""": 1,
            """top_p""": 0.7,
            """top_k""": 10,
            """temperature""": 0.7,
        }
        _lowercase	       =14

        _lowercase	       =AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'					)
        _lowercase	       ="""Hello, my dog is cute and"""
        _lowercase	       =tokenizer(snake_case_							,   return_tensors='tf'					)
        _lowercase	       =TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2'					)

        _lowercase	       =638
        # forces the generation to happen on CPU, to avoid GPU-related quirks
        with tf.device(':/CPU:0'					):
              tf.random.set_seed(0					)
              _lowercase	       =model.generate(**snake_case_							,   eos_token_id=snake_case_							,   **snake_case_					)
        self.assertTrue(expectation == len(generated_tokens[0]					)					)

        _lowercase	       =[638, 198]
        with tf.device(':/CPU:0'					):
              tf.random.set_seed(0					)
              _lowercase	       =model.generate(**snake_case_							,   eos_token_id=snake_case_							,   **snake_case_					)
        self.assertTrue(expectation == len(generated_tokens[0]					)					)




  def   A__     (				self					)  -> Optional[int]:
        '''simple docstring'''






        _lowercase	       =AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bart'					)
        _lowercase	       ="""Hugging Face is a technology company based in New York and Paris."""
        _lowercase	       =bart_tokenizer(snake_case_							,   return_tensors='tf'					).input_ids
        _lowercase	       =TFBartForConditionalGeneration.from_pretrained('hf-internal-testing/tiny-random-bart'					)
        _lowercase	       =bart_model.generate(snake_case_					).numpy()





        class  __lowerCAmelCase     (					_a							):




          def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase=None							,   **lowerCAmelCase					)  -> Optional[Any]:
                '''simple docstring'''






                return super().call(snake_case_							,   **snake_case_					)
        _lowercase	       =FakeBart.from_pretrained('hf-internal-testing/tiny-random-bart'					)
        _lowercase	       =bart_model.generate(snake_case_							,   foo='bar'					).numpy()
        self.assertTrue(np.array_equal(snake_case_							,   snake_case_					)					)





        class  __lowerCAmelCase     (					bart_model.model.encoder.__class__							):




          def   A__     (				self							,   lowerCAmelCase							,   **lowerCAmelCase					)  -> int:
                '''simple docstring'''






                return super().call(snake_case_							,   **snake_case_					)
        _lowercase	       =FakeEncoder(bart_model.config							,   bart_model.model.shared					)
        _lowercase	       =fake_encoder

        # Normal generation still works (the output will be different because the encoder weights are different)
        _lowercase	       =bart_model.generate(snake_case_					).numpy()
        with self.assertRaises(snake_case_					):
              # FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
              bart_model.generate(snake_case_							,   foo='bar'					)







'''simple docstring'''





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     UpperCAmelCase_       (       _a		,		unittest.TestCase	):
    """simple docstring"""







    lowercase					=		GPTSanJapaneseTokenizer
    lowercase					=		False
    lowercase					=		{"do_clean_text": False, "add_prefix_space": False}




    def lowerCamelCase       (    self					:     str						):
          super().setUp()

          # fmt: off
          snake_case__  :    Optional[Any]    						=      ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
          # fmt: on
          snake_case__  :    int    						=      {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
          snake_case__  :    List[Any]    						=      {"""unk_token""": """<unk>"""}

          snake_case__  :    Optional[Any]    						=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
          snake_case__  :    Dict    						=      os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""emoji_file"""]						)
          with open(self.vocab_file ,     """w""" ,     encoding="""utf-8"""						) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)
          with open(self.emoji_file ,     """w"""						) as emoji_writer:
                emoji_writer.write(json.dumps(snake_case_						)						)




    def lowerCamelCase       (    self					:     Any ,     **snake_case_					:     Union[str, Any]						):
          kwargs.update(self.special_tokens_map						)
          return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname ,     **snake_case_						)




    def lowerCamelCase       (    self					:     Any ,     snake_case_					:     str						):
          snake_case__  :    Union[str, Any]    						=      """こんにちは、世界。 \nこんばんは、㔺界。😀"""
          snake_case__  :    List[str]    						=      """こんにちは、世界。 \nこんばんは、世界。😀"""
          return input_text, output_text




    def lowerCamelCase       (    self					:     Any ,     snake_case_					:     Dict						):
          snake_case__						,				snake_case__  :    int    						=      self.get_input_output_texts(snake_case_						)
          snake_case__  :    int    						=      tokenizer.encode(snake_case_ ,     add_special_tokens=snake_case_						)
          snake_case__  :    List[str]    						=      tokenizer.decode(snake_case_ ,     clean_up_tokenization_spaces=snake_case_						)
          return text, ids




    def lowerCamelCase       (    self					:     Optional[Any]						):
          pass  # TODO add if relevant




    def lowerCamelCase       (    self					:     Union[str, Any]						):
          pass  # TODO add if relevant




    def lowerCamelCase       (    self					:     List[str]						):
          pass  # TODO add if relevant




    def lowerCamelCase       (    self					:     Dict						):
          snake_case__  :    Optional[Any]    						=      self.get_tokenizer()

          # Testing tokenization
          snake_case__  :    int    						=      """こんにちは、世界。　こんばんは、㔺界。"""
          snake_case__  :    Optional[int]    						=      ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
          snake_case__  :    Dict    						=      tokenizer.tokenize(snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)

          # Testing conversion to ids without special tokens
          snake_case__  :    Union[str, Any]    						=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
          snake_case__  :    List[Any]    						=      tokenizer.convert_tokens_to_ids(snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)

          # Testing conversion to ids with special tokens
          snake_case__  :    Union[str, Any]    						=      tokens + [tokenizer.unk_token]
          snake_case__  :    Dict    						=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
          snake_case__  :    Any    						=      tokenizer.convert_tokens_to_ids(snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)




    def lowerCamelCase       (    self					:     Optional[Any]						):
          snake_case__  :    Union[str, Any]    						=      self.get_tokenizer()

          # Testing tokenization
          snake_case__  :    Union[str, Any]    						=      """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
          snake_case__  :    Optional[int]    						=      """こんにちは、、、、世界。こんばんは、、、、世界。"""
          snake_case__  :    Any    						=      tokenizer.encode(snake_case_						)
          snake_case__  :    int    						=      tokenizer.decode(snake_case_						)
          self.assertEqual(snake_case_ ,     snake_case_						)




    @slow
    def lowerCamelCase       (    self					:     Union[str, Any]						):
          snake_case__  :    Optional[Any]    						=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""						)

          # Testing tokenization
          snake_case__  :    Tuple    						=      """こんにちは、世界。"""
          snake_case__  :    Optional[Any]    						=      """こんばんは、㔺界。😀"""
          snake_case__  :    List[str]    						=      """こんにちは、世界。こんばんは、世界。😀"""
          snake_case__  :    Dict    						=      tokenizer.encode(prefix_text + input_text						)
          snake_case__  :    Dict    						=      tokenizer.encode("""""" ,     prefix_text=prefix_text + input_text						)
          snake_case__  :    int    						=      tokenizer.encode(snake_case_ ,     prefix_text=snake_case_						)
          snake_case__  :    Optional[Any]    						=      tokenizer.decode(snake_case_						)
          snake_case__  :    Union[str, Any]    						=      tokenizer.decode(snake_case_						)
          snake_case__  :    str    						=      tokenizer.decode(snake_case_						)
          self.assertEqual(snake_case_ ,     snake_case_						)
          self.assertEqual(snake_case_ ,     snake_case_						)
          self.assertEqual(snake_case_ ,     snake_case_						)




    @slow
    def lowerCamelCase       (    self					:     Union[str, Any]						):
          snake_case__  :    Optional[Any]    						=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""						)

          # Testing tokenization
          snake_case__  :    Dict    						=      """こんにちは、世界。"""
          snake_case__  :    Optional[int]    						=      """こんばんは、㔺界。😀"""

          snake_case__  :    Any    						=      len(tokenizer.encode(snake_case_						)						) - 2
          snake_case__  :    Optional[int]    						=      len(tokenizer.encode(snake_case_						)						) - 2

          snake_case__  :    List[str]    						=      [1] + [0] * (len_prefix + len_text + 1)
          snake_case__  :    Optional[int]    						=      [1] * (len_prefix + len_text + 1) + [0]
          snake_case__  :    int    						=      [1] + [1] * (len_prefix) + [0] * (len_text + 1)

          snake_case__  :    Any    						=      tokenizer(prefix_text + input_text						).token_type_ids
          snake_case__  :    str    						=      tokenizer("""""" ,     prefix_text=prefix_text + input_text						).token_type_ids
          snake_case__  :    Optional[Any]    						=      tokenizer(snake_case_ ,     prefix_text=snake_case_						).token_type_ids
          self.assertListEqual(snake_case_ ,     snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)
          self.assertListEqual(snake_case_ ,     snake_case_						)




    @slow
    def lowerCamelCase       (    self					:     Optional[int]						):
          snake_case__  :    Optional[Any]    						=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""						)

          snake_case__  :    Union[str, Any]    						=      tokenizer.encode("""あンいワ"""						)
          snake_case__  :    int    						=      tokenizer.encode("""""" ,     prefix_text="""あンいワ"""						)
          snake_case__  :    Dict    						=      tokenizer.encode("""いワ""" ,     prefix_text="""あン"""						)

          self.assertEqual(tokenizer.decode(snake_case_						) ,     tokenizer.decode(snake_case_						)						)
          self.assertEqual(tokenizer.decode(snake_case_						) ,     tokenizer.decode(snake_case_						)						)
          self.assertNotEqual(snake_case_ ,     snake_case_						)
          self.assertNotEqual(snake_case_ ,     snake_case_						)
          self.assertEqual(x_token_a[1] ,     x_token_a[-1]						)  # SEG token
          self.assertEqual(x_token_a[1] ,     x_token_a[3]						)  # SEG token




    @slow
    def lowerCamelCase       (    self					:     Any						):
          snake_case__  :    Optional[int]    						=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""						)

          snake_case__  :    int    						=      [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
          snake_case__  :    Optional[Any]    						=      tokenizer(snake_case_ ,     padding=snake_case_						)
          snake_case__  :    Tuple    						=      tokenizer.batch_encode_plus(snake_case_ ,     padding=snake_case_						)

          # fmt: off
          snake_case__  :    Optional[Any]    						=      [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
          snake_case__  :    Optional[Any]    						=      [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
          snake_case__  :    Optional[Any]    						=      [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
          # fmt: on
          self.assertListEqual(x_token.input_ids ,     snake_case_						)
          self.assertListEqual(x_token.token_type_ids ,     snake_case_						)
          self.assertListEqual(x_token.attention_mask ,     snake_case_						)
          self.assertListEqual(x_token_a.input_ids ,     snake_case_						)
          self.assertListEqual(x_token_a.token_type_ids ,     snake_case_						)
          self.assertListEqual(x_token_a.attention_mask ,     snake_case_						)




    def lowerCamelCase       (    self					:     Any						):
          # Intentionally convert some words to accommodate character fluctuations unique to Japanese
          pass




    def lowerCamelCase       (    self					:     List[str]						):
          # tokenizer has no padding token
          pass
