
"""simple docstring"""


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class     A_	(     A_      ,					unittest.TestCase   ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_	      =   CLIPTokenizer
							SCREAMING_SNAKE_CASE_	      =   CLIPTokenizerFast
							SCREAMING_SNAKE_CASE_	      =   True
							SCREAMING_SNAKE_CASE_	      =   {}
							SCREAMING_SNAKE_CASE_	      =   False





							def       UpperCAmelCase__							(  self  :List[Any]       ):





										"""simple docstring"""

										super().setUp()

										# fmt: off
										lowerCamelCase__	:     Dict              =['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
										# fmt: on
										lowerCamelCase__	:     Union[str, Any]              =dict(zip(_snake_case     ,							range(len(_snake_case       )       )       )       )
										lowerCamelCase__	:     Union[str, Any]              =['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
										lowerCamelCase__	:     List[str]              ={'''unk_token''': '''<unk>'''}

										lowerCamelCase__	:     List[Any]              =os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['vocab_file']       )
										lowerCamelCase__	:     List[str]              =os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['merges_file']       )
										with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'       ) as fp:
													fp.write(json.dumps(_snake_case       ) + '\n'       )
										with open(self.merges_file     ,							'w'     ,							encoding='utf-8'       ) as fp:
													fp.write('\n'.join(_snake_case       )       )





							def       UpperCAmelCase__							(  self  :Any     ,							**lowerCamelCase_  :Any       ):





										"""simple docstring"""

										kwargs.update(self.special_tokens_map       )
										return CLIPTokenizer.from_pretrained(self.tmpdirname     ,							**_snake_case       )





							def       UpperCAmelCase__							(  self  :Optional[Any]     ,							**lowerCamelCase_  :List[str]       ):





										"""simple docstring"""

										kwargs.update(self.special_tokens_map       )
										return CLIPTokenizerFast.from_pretrained(self.tmpdirname     ,							**_snake_case       )





							def       UpperCAmelCase__							(  self  :Optional[int]     ,							lowerCamelCase_  :List[str]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Dict              ='''lower newer'''
										lowerCamelCase__	:     Optional[Any]              ='''lower newer'''
										return input_text, output_text





							def       UpperCAmelCase__							(  self  :List[Any]       ):





										"""simple docstring"""

										lowerCamelCase__	:     Union[str, Any]              =CLIPTokenizer(self.vocab_file     ,							self.merges_file     ,							**self.special_tokens_map       )
										lowerCamelCase__	:     Dict              ='''lower newer'''
										lowerCamelCase__	:     Union[str, Any]              =['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
										lowerCamelCase__	:     Tuple              =tokenizer.tokenize(_snake_case       )
										self.assertListEqual(_snake_case     ,							_snake_case       )

										lowerCamelCase__	:     List[str]              =tokens + [tokenizer.unk_token]
										lowerCamelCase__	:     Optional[Any]              =[10, 2, 16, 9, 3, 2, 16, 20]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case       )     ,							_snake_case       )





							@require_ftfy
							def       UpperCAmelCase__							(  self  :Tuple       ):





										"""simple docstring"""

										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																lowerCamelCase__	:     List[str]              =self.tokenizer_class.from_pretrained(_snake_case     ,							**_snake_case       )
																lowerCamelCase__	:     Dict              =self.rust_tokenizer_class.from_pretrained(_snake_case     ,							**_snake_case       )

																lowerCamelCase__	:     Union[str, Any]              ='''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																lowerCamelCase__	:     str              =tokenizer_s.tokenize(_snake_case       )
																lowerCamelCase__	:     Optional[Any]              =tokenizer_r.tokenize(_snake_case       )

																self.assertListEqual(_snake_case     ,							_snake_case       )

																# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																# with Tilde) encoded in 2 different ways
																lowerCamelCase__	:     Optional[Any]              ='''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																lowerCamelCase__	:     int              =tokenizer_s.tokenize(_snake_case       )
																lowerCamelCase__	:     Union[str, Any]              =tokenizer_r.tokenize(_snake_case       )

																self.assertListEqual(_snake_case     ,							_snake_case       )

																# Test that the tokenization is identical on unicode of space type
																lowerCamelCase__	:     Optional[int]              =[
																    '''\u0009''',  # (horizontal tab, '\t')
																    '''\u000B''',  # (vertical tab)
																    '''\u000C''',  # (form feed)
																    '''\u0020''',  # (space, ' ')
																    '''\u200E''',  # (left-to-right mark):w
																    '''\u200F''',  # (right-to-left mark)
																]
																for unicode_seq in spaces_unicodes:
																			lowerCamelCase__	:     Tuple              =tokenizer_s.tokenize(_snake_case       )
																			lowerCamelCase__	:     Optional[int]              =tokenizer_r.tokenize(_snake_case       )

																			self.assertListEqual(_snake_case     ,							_snake_case       )

																# Test that the tokenization is identical on unicode of line break type
																lowerCamelCase__	:     Tuple              =[
																    '''\u000A''',  # (line feed, '\n')
																    '''\r\n''',  # (carriage return and line feed, '\r\n')
																    '''\u000D''',  # (carriage return, '\r')
																    '''\r''',  # (carriage return, '\r')
																    '''\u000D''',  # (carriage return, '\r')
																    '''\u2028''',  # (line separator)
																    '''\u2029''',  # (paragraph separator)
																    # "\u0085", # (next line)
																]

																# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																# space (and thus into an empty list).

																for unicode_seq in line_break_unicodes:
																			lowerCamelCase__	:     Dict              =tokenizer_s.tokenize(_snake_case       )
																			lowerCamelCase__	:     Tuple              =tokenizer_r.tokenize(_snake_case       )

																			self.assertListEqual(_snake_case     ,							_snake_case       )





							def       UpperCAmelCase__							(  self  :Union[str, Any]       ):





										"""simple docstring"""

										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																lowerCamelCase__	:     Union[str, Any]              ='''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																lowerCamelCase__	:     List[Any]              =f"""{text_of_1_token} {text_of_1_token}"""

																lowerCamelCase__	:     Dict              =self.rust_tokenizer_class.from_pretrained(
																    _snake_case     ,							use_fast=_snake_case     ,							)
																lowerCamelCase__	:     Optional[int]              =tokenizer_r(_snake_case     ,							return_offsets_mapping=_snake_case     ,							add_special_tokens=_snake_case       )
																self.assertEqual(encoding.offset_mapping[0]     ,							(0, len(_snake_case       ))       )
																self.assertEqual(
																    encoding.offset_mapping[1]     ,							(len(_snake_case       ) + 1, len(_snake_case       ) + 1 + len(_snake_case       ))     ,							)

																lowerCamelCase__	:     Tuple              =f""" {text}"""

																lowerCamelCase__	:     Any              =self.rust_tokenizer_class.from_pretrained(
																    _snake_case     ,							use_fast=_snake_case     ,							)
																lowerCamelCase__	:     Dict              =tokenizer_r(_snake_case     ,							return_offsets_mapping=_snake_case     ,							add_special_tokens=_snake_case       )
																self.assertEqual(encoding.offset_mapping[0]     ,							(1, 1 + len(_snake_case       ))       )
																self.assertEqual(
																    encoding.offset_mapping[1]     ,							(1 + len(_snake_case       ) + 1, 1 + len(_snake_case       ) + 1 + len(_snake_case       ))     ,							)





							def       UpperCAmelCase__							(  self  :Any       ):





										"""simple docstring"""

										with self.assertRaises(_snake_case       ) as context:
													self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'       )

										self.assertTrue(
										    context.exception.args[0].startswith(
										        'The `backend_tokenizer` provided does not match the expected format.'       )       )





							@require_ftfy
							def       UpperCAmelCase__							(  self  :Any       ):





										"""simple docstring"""

										super().test_tokenization_python_rust_equals()





							def       UpperCAmelCase__							(  self  :str       ):





										"""simple docstring"""

										pass




"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase_   						=   {'configuration_xglm': ['XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XGLMConfig']}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   ['XGLMTokenizer']

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   ['XGLMTokenizerFast']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'XGLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'XGLMForCausalLM',
							    'XGLMModel',
							    'XGLMPreTrainedModel',
							]


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'FlaxXGLMForCausalLM',
							    'FlaxXGLMModel',
							    'FlaxXGLMPreTrainedModel',
							]


try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCAmelCase_   						=   [
							    'TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFXGLMForCausalLM',
							    'TFXGLMModel',
							    'TFXGLMPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm import XGLMTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm_fast import XGLMTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_xglm import (
														    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFXGLMForCausalLM,
														    TFXGLMModel,
														    TFXGLMPreTrainedModel,
														)


else:
							import sys

							lowerCAmelCase_   						=   _LazyModule(__name__, globals()['__file__'], _import_structure)
