
'''simple docstring'''




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class  A				(							__snake_case    ,			unittest.TestCase						):
     __magic_name__						=		CLIPTokenizer
     __magic_name__						=		CLIPTokenizerFast
     __magic_name__						=		True
     __magic_name__						=		{}
     __magic_name__						=		False





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         super().setUp()

         # fmt: off
         A       :   List[Any]					=						['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
         # fmt: on
         A       :   Tuple					=						dict(zip(SCREAMING_SNAKE_CASE						,			range(len(SCREAMING_SNAKE_CASE	)	)	)	)
         A       :   Any					=						['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
         A       :   List[Any]					=						{'''unk_token''': '''<unk>'''}

         A       :   List[str]					=						os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES['''vocab_file''']	)
         A       :   int					=						os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES['''merges_file''']	)
         with open(self.vocab_file						,			'''w'''						,			encoding='''utf-8'''	) as fp:
             fp.write(json.dumps(SCREAMING_SNAKE_CASE	) + '''\n'''	)
         with open(self.merges_file						,			'''w'''						,			encoding='''utf-8'''	) as fp:
             fp.write('''\n'''.join(SCREAMING_SNAKE_CASE	)	)





     def 				__lowerCAmelCase							(							self						,			**SCREAMING_SNAKE_CASE	) ->				Union[str, Any]:






         """simple docstring"""






         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizer.from_pretrained(self.tmpdirname						,			**SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			**SCREAMING_SNAKE_CASE	) ->				Tuple:






         """simple docstring"""






         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,			**SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   str					=						'''lower newer'''
         A       :   int					=						'''lower newer'''
         return input_text, output_text





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   Tuple					=						CLIPTokenizer(self.vocab_file						,			self.merges_file						,			**self.special_tokens_map	)
         A       :   Optional[Any]					=						'''lower newer'''
         A       :   int					=						['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
         A       :   List[str]					=						tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
         self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

         A       :   List[Any]					=						tokens + [tokenizer.unk_token]
         A       :   Any					=						[10, 2, 16, 9, 3, 2, 16, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)





     @require_ftfy
     def 				__lowerCAmelCase							(							self	) ->				List[Any]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                 A       :   int					=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)
                 A       :   List[Any]					=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)

                 A       :   List[str]					=						'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                 A       :   Any					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                 A       :   List[Any]					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                 # with Tilde) encoded in 2 different ways
                 A       :   List[Any]					=						'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                 A       :   Union[str, Any]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                 A       :   Dict					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on unicode of space type
                 A       :   Tuple					=						[
                     '''\u0009''',  # (horizontal tab, '\t')
                     '''\u000B''',  # (vertical tab)
                     '''\u000C''',  # (form feed)
                     '''\u0020''',  # (space, ' ')
                     '''\u200E''',  # (left-to-right mark):w
                     '''\u200F''',  # (right-to-left mark)
                 ]
                 for unicode_seq in spaces_unicodes:
                     A       :   List[str]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                     A       :   List[Any]					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                     self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on unicode of line break type
                 A       :   int					=						[
                     '''\u000A''',  # (line feed, '\n')
                     '''\r\n''',  # (carriage return and line feed, '\r\n')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\r''',  # (carriage return, '\r')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\u2028''',  # (line separator)
                     '''\u2029''',  # (paragraph separator)
                     # "\u0085", # (next line)
                 ]

                 # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                 # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                 # space (and thus into an empty list).

                 for unicode_seq in line_break_unicodes:
                     A       :   List[str]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                     A       :   Tuple					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                     self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                 A       :   int					=						'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                 A       :   List[Any]					=						F'{text_of_1_token} {text_of_1_token}'

                 A       :   Optional[Any]					=						self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE						,			use_fast=SCREAMING_SNAKE_CASE						,			)
                 A       :   Optional[int]					=						tokenizer_r(SCREAMING_SNAKE_CASE						,			return_offsets_mapping=SCREAMING_SNAKE_CASE						,			add_special_tokens=SCREAMING_SNAKE_CASE	)
                 self.assertEqual(encoding.offset_mapping[0]						,			(0, len(SCREAMING_SNAKE_CASE	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]						,			(len(SCREAMING_SNAKE_CASE	) + 1, len(SCREAMING_SNAKE_CASE	) + 1 + len(SCREAMING_SNAKE_CASE	))						,			)

                 A       :   Any					=						F' {text}'

                 A       :   Dict					=						self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE						,			use_fast=SCREAMING_SNAKE_CASE						,			)
                 A       :   int					=						tokenizer_r(SCREAMING_SNAKE_CASE						,			return_offsets_mapping=SCREAMING_SNAKE_CASE						,			add_special_tokens=SCREAMING_SNAKE_CASE	)
                 self.assertEqual(encoding.offset_mapping[0]						,			(1, 1 + len(SCREAMING_SNAKE_CASE	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]						,			(1 + len(SCREAMING_SNAKE_CASE	) + 1, 1 + len(SCREAMING_SNAKE_CASE	) + 1 + len(SCREAMING_SNAKE_CASE	))						,			)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         with self.assertRaises(SCREAMING_SNAKE_CASE	) as context:
             self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''	)

         self.assertTrue(
             context.exception.args[0].startswith(
                 '''The `backend_tokenizer` provided does not match the expected format.'''	)	)





     @require_ftfy
     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         super().test_tokenization_python_rust_equals()





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         pass






'''simple docstring'''




import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Tuple:






         """simple docstring"""






         A       :   Union[str, Any]					=						[]
         A       :   Union[str, Any]					=						[]

         for i in range(self.num_layers	):
             A       :   Any					=						self.in_channels if i == 0 else self.out_channels

             A       :   Optional[Any]					=						FlaxResnetBlockaD(
                 in_channels=SCREAMING_SNAKE_CASE						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

             A       :   Optional[int]					=						FlaxTransformeraDModel(
                 in_channels=self.out_channels						,			n_heads=self.num_attention_heads						,			d_head=self.out_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			only_cross_attention=self.only_cross_attention						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

         A       :   Union[str, Any]					=						resnets
         A       :   Union[str, Any]					=						attentions

         if self.add_downsample:
             A       :   int					=						FlaxDownsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						()

         for resnet, attn in zip(self.resnets						,			self.attentions	):
             A       :   int					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Dict					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         if self.add_downsample:
             A       :   Optional[Any]					=						self.downsamplers_a(SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         return hidden_states, output_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         A       :   Optional[Any]					=						[]

         for i in range(self.num_layers	):
             A       :   Optional[Any]					=						self.in_channels if i == 0 else self.out_channels

             A       :   List[str]					=						FlaxResnetBlockaD(
                 in_channels=SCREAMING_SNAKE_CASE						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)
         A       :   Dict					=						resnets

         if self.add_downsample:
             A       :   Dict					=						FlaxDownsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Optional[Any]:






         """simple docstring"""






         A       :   str					=						()

         for resnet in self.resnets:
             A       :   Optional[int]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         if self.add_downsample:
             A       :   Optional[int]					=						self.downsamplers_a(SCREAMING_SNAKE_CASE	)
             output_states += (hidden_states,)

         return hidden_states, output_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Tuple:






         """simple docstring"""






         A       :   Optional[Any]					=						[]
         A       :   Optional[int]					=						[]

         for i in range(self.num_layers	):
             A       :   str					=						self.in_channels if (i == self.num_layers - 1) else self.out_channels
             A       :   Dict					=						self.prev_output_channel if i == 0 else self.out_channels

             A       :   List[str]					=						FlaxResnetBlockaD(
                 in_channels=resnet_in_channels + res_skip_channels						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

             A       :   int					=						FlaxTransformeraDModel(
                 in_channels=self.out_channels						,			n_heads=self.num_attention_heads						,			d_head=self.out_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			only_cross_attention=self.only_cross_attention						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

         A       :   Dict					=						resnets
         A       :   Optional[Any]					=						attentions

         if self.add_upsample:
             A       :   Optional[int]					=						FlaxUpsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Optional[int]:






         """simple docstring"""






         for resnet, attn in zip(self.resnets						,			self.attentions	):
             # pop res hidden states
             A       :   List[str]					=						res_hidden_states_tuple[-1]
             A       :   int					=						res_hidden_states_tuple[:-1]
             A       :   List[str]					=						jnp.concatenate((hidden_states, res_hidden_states)						,			axis=-1	)

             A       :   Union[str, Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Tuple					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         if self.add_upsample:
             A       :   Dict					=						self.upsamplers_a(SCREAMING_SNAKE_CASE	)

         return hidden_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		True
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   int					=						[]

         for i in range(self.num_layers	):
             A       :   List[Any]					=						self.in_channels if (i == self.num_layers - 1) else self.out_channels
             A       :   List[str]					=						self.prev_output_channel if i == 0 else self.out_channels

             A       :   str					=						FlaxResnetBlockaD(
                 in_channels=resnet_in_channels + res_skip_channels						,			out_channels=self.out_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

         A       :   List[Any]					=						resnets

         if self.add_upsample:
             A       :   Optional[Any]					=						FlaxUpsampleaD(self.out_channels						,			dtype=self.dtype	)





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Tuple:






         """simple docstring"""






         for resnet in self.resnets:
             # pop res hidden states
             A       :   Optional[int]					=						res_hidden_states_tuple[-1]
             A       :   Optional[Any]					=						res_hidden_states_tuple[:-1]
             A       :   List[Any]					=						jnp.concatenate((hidden_states, res_hidden_states)						,			axis=-1	)

             A       :   Optional[Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         if self.add_upsample:
             A       :   List[str]					=						self.upsamplers_a(SCREAMING_SNAKE_CASE	)

         return hidden_states





class  A				(							nn.Module						):
     __magic_name__						=		42
     __magic_name__						=		0.0
     __magic_name__						=		1
     __magic_name__						=		1
     __magic_name__						=		False
     __magic_name__						=		False
     __magic_name__						=		jnp.floataa





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   str					=						[
             FlaxResnetBlockaD(
                 in_channels=self.in_channels						,			out_channels=self.in_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
         ]

         A       :   List[Any]					=						[]

         for _ in range(self.num_layers	):
             A       :   int					=						FlaxTransformeraDModel(
                 in_channels=self.in_channels						,			n_heads=self.num_attention_heads						,			d_head=self.in_channels // self.num_attention_heads						,			depth=1						,			use_linear_projection=self.use_linear_projection						,			use_memory_efficient_attention=self.use_memory_efficient_attention						,			dtype=self.dtype						,			)
             attentions.append(SCREAMING_SNAKE_CASE	)

             A       :   Union[str, Any]					=						FlaxResnetBlockaD(
                 in_channels=self.in_channels						,			out_channels=self.in_channels						,			dropout_prob=self.dropout						,			dtype=self.dtype						,			)
             resnets.append(SCREAMING_SNAKE_CASE	)

         A       :   List[str]					=						resnets
         A       :   List[str]					=						attentions





     def __call__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=True	) ->				Dict:






         """simple docstring"""






         A       :   Optional[Any]					=						self.resnets[0](SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)
         for attn, resnet in zip(self.attentions						,			self.resnets[1:]	):
             A       :   Optional[int]					=						attn(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)
             A       :   Union[str, Any]					=						resnet(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE						,			deterministic=SCREAMING_SNAKE_CASE	)

         return hidden_states




