'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 				_lowercase     (				A__  ,							unittest.TestCase			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       str      	=					CLIPTokenizer
 _SCREAMING_SNAKE_CASE     :       Optional[Any]      	=					CLIPTokenizerFast
 _SCREAMING_SNAKE_CASE     :       Tuple      	=					True
 _SCREAMING_SNAKE_CASE     :       List[str]      	=					{}
 _SCREAMING_SNAKE_CASE     :       Dict      	=					False
 def a					(      self   :	Any							)  ->    Optional[int]:
  super().setUp()

  # fmt: off
  __lowerCAmelCase											=						['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
  # fmt: on
  __lowerCAmelCase											=						dict(zip(__snake_case      ,       range(len(__snake_case							)							)							)							)
  __lowerCAmelCase											=						['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
  __lowerCAmelCase											=						{'''unk_token''': '''<unk>'''}

  __lowerCAmelCase											=						os.path.join(self.tmpdirname      ,       VOCAB_FILES_NAMES["""vocab_file"""]							)
  __lowerCAmelCase											=						os.path.join(self.tmpdirname      ,       VOCAB_FILES_NAMES["""merges_file"""]							)
  with open(self.vocab_file      ,       """w"""      ,       encoding="""utf-8"""							) as fp:
   fp.write(json.dumps(__snake_case							) + """\n"""							)
  with open(self.merges_file      ,       """w"""      ,       encoding="""utf-8"""							) as fp:
   fp.write("""\n""".join(__snake_case							)							)
 def a					(      self   :	Optional[int]      ,       **SCREAMING_SNAKE_CASE__   :	Optional[int]							)  ->    Optional[int]:
  kwargs.update(self.special_tokens_map							)
  return CLIPTokenizer.from_pretrained(self.tmpdirname      ,       **__snake_case							)
 def a					(      self   :	List[Any]      ,       **SCREAMING_SNAKE_CASE__   :	Any							)  ->    List[Any]:
  kwargs.update(self.special_tokens_map							)
  return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,       **__snake_case							)
 def a					(      self   :	List[str]      ,       SCREAMING_SNAKE_CASE__   :	Tuple							)  ->    int:
  __lowerCAmelCase											=						'''lower newer'''
  __lowerCAmelCase											=						'''lower newer'''
  return input_text, output_text
 def a					(      self   :	Dict							)  ->    List[Any]:
  __lowerCAmelCase											=						CLIPTokenizer(self.vocab_file      ,       self.merges_file      ,       **self.special_tokens_map							)
  __lowerCAmelCase											=						'''lower newer'''
  __lowerCAmelCase											=						['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
  __lowerCAmelCase											=						tokenizer.tokenize(__snake_case							)
  self.assertListEqual(__snake_case      ,       __snake_case							)

  __lowerCAmelCase											=						tokens + [tokenizer.unk_token]
  __lowerCAmelCase											=						[10, 2, 16, 9, 3, 2, 16, 20]
  self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case							)      ,       __snake_case							)
 @require_ftfy
 def a					(      self   :	str							)  ->    str:
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
    __lowerCAmelCase											=						self.tokenizer_class.from_pretrained(__snake_case      ,       **__snake_case							)
    __lowerCAmelCase											=						self.rust_tokenizer_class.from_pretrained(__snake_case      ,       **__snake_case							)

    __lowerCAmelCase											=						'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
    __lowerCAmelCase											=						tokenizer_s.tokenize(__snake_case							)
    __lowerCAmelCase											=						tokenizer_r.tokenize(__snake_case							)

    self.assertListEqual(__snake_case      ,       __snake_case							)

    # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
    # with Tilde) encoded in 2 different ways
    __lowerCAmelCase											=						'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
    __lowerCAmelCase											=						tokenizer_s.tokenize(__snake_case							)
    __lowerCAmelCase											=						tokenizer_r.tokenize(__snake_case							)

    self.assertListEqual(__snake_case      ,       __snake_case							)

    # Test that the tokenization is identical on unicode of space type
    __lowerCAmelCase											=						[
        '''\u0009''',  # (horizontal tab, '\t')
        '''\u000B''',  # (vertical tab)
        '''\u000C''',  # (form feed)
        '''\u0020''',  # (space, ' ')
        '''\u200E''',  # (left-to-right mark):w
        '''\u200F''',  # (right-to-left mark)
    ]
    for unicode_seq in spaces_unicodes:
     __lowerCAmelCase											=						tokenizer_s.tokenize(__snake_case							)
     __lowerCAmelCase											=						tokenizer_r.tokenize(__snake_case							)

     self.assertListEqual(__snake_case      ,       __snake_case							)

    # Test that the tokenization is identical on unicode of line break type
    __lowerCAmelCase											=						[
        '''\u000A''',  # (line feed, '\n')
        '''\r\n''',  # (carriage return and line feed, '\r\n')
        '''\u000D''',  # (carriage return, '\r')
        '''\r''',  # (carriage return, '\r')
        '''\u000D''',  # (carriage return, '\r')
        '''\u2028''',  # (line separator)
        '''\u2029''',  # (paragraph separator)
        # "\u0085", # (next line)
    ]

    # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
    # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
    # space (and thus into an empty list).

    for unicode_seq in line_break_unicodes:
     __lowerCAmelCase											=						tokenizer_s.tokenize(__snake_case							)
     __lowerCAmelCase											=						tokenizer_r.tokenize(__snake_case							)

     self.assertListEqual(__snake_case      ,       __snake_case							)
 def a					(      self   :	Tuple							)  ->    Optional[int]:
  # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
    __lowerCAmelCase											=						'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
    __lowerCAmelCase											=						f"""{text_of_1_token} {text_of_1_token}"""

    __lowerCAmelCase											=						self.rust_tokenizer_class.from_pretrained(
        __snake_case      ,       use_fast=__snake_case      ,       )
    __lowerCAmelCase											=						tokenizer_r(__snake_case      ,       return_offsets_mapping=__snake_case      ,       add_special_tokens=__snake_case							)
    self.assertEqual(encoding.offset_mapping[0]      ,       (0, len(__snake_case							))							)
    self.assertEqual(
        encoding.offset_mapping[1]      ,       (len(__snake_case							) + 1, len(__snake_case							) + 1 + len(__snake_case							))      ,       )

    __lowerCAmelCase											=						f""" {text}"""

    __lowerCAmelCase											=						self.rust_tokenizer_class.from_pretrained(
        __snake_case      ,       use_fast=__snake_case      ,       )
    __lowerCAmelCase											=						tokenizer_r(__snake_case      ,       return_offsets_mapping=__snake_case      ,       add_special_tokens=__snake_case							)
    self.assertEqual(encoding.offset_mapping[0]      ,       (1, 1 + len(__snake_case							))							)
    self.assertEqual(
        encoding.offset_mapping[1]      ,       (1 + len(__snake_case							) + 1, 1 + len(__snake_case							) + 1 + len(__snake_case							))      ,       )
 def a					(      self   :	str							)  ->    str:
  # Test related to the breaking change introduced in transformers v4.17.0
  # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
  with self.assertRaises(__snake_case							) as context:
   self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""							)

  self.assertTrue(
      context.exception.args[0].startswith(
          """The `backend_tokenizer` provided does not match the expected format."""							)							)
 @require_ftfy
 def a					(      self   :	str							)  ->    Dict:
  super().test_tokenization_python_rust_equals()





 def a					(      self   :	Optional[Any]							)  ->    Optional[Any]:
  # CLIP always lower cases letters
  pass











'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__:   str								=			{
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   int								=			["LxmertTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   Union[str, Any]								=			[
						    "LxmertEncoder",
						    "LxmertForPreTraining",
						    "LxmertForQuestionAnswering",
						    "LxmertModel",
						    "LxmertPreTrainedModel",
						    "LxmertVisualFeatureEncoder",
						    "LxmertXLayer",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__:   int								=			[
						    "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "TFLxmertForPreTraining",
						    "TFLxmertMainLayer",
						    "TFLxmertModel",
						    "TFLxmertPreTrainedModel",
						    "TFLxmertVisualFeatureEncoder",
						]


if TYPE_CHECKING:
						from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
						from .tokenization_lxmert import LxmertTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_lxmert_fast import LxmertTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_lxmert import (
												    LxmertEncoder,
												    LxmertForPreTraining,
												    LxmertForQuestionAnswering,
												    LxmertModel,
												    LxmertPreTrainedModel,
												    LxmertVisualFeatureEncoder,
												    LxmertXLayer,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_lxmert import (
												    TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFLxmertForPreTraining,
												    TFLxmertMainLayer,
												    TFLxmertModel,
												    TFLxmertPreTrainedModel,
												    TFLxmertVisualFeatureEncoder,
												)

else:
						import sys

						UpperCamelCase__:   Tuple								=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





