


'''simple docstring'''

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			UpperCAmelCase			(							SCREAMING_SNAKE_CASE__		,      unittest.TestCase):
 _lowerCamelCase			:       Union[str, Any]            =       ElectraTokenizer
 _lowerCamelCase			:       Dict            =       ElectraTokenizerFast
 _lowerCamelCase			:       Optional[Any]            =       True
 _lowerCamelCase			:       str            =       True
 _lowerCamelCase			:       Union[str, Any]            =       filter_non_english

 def 		lowercase_    (			self			:						Tuple       ):





    """simple docstring"""



    super().setUp()

    UpperCamelCase__			    =    [
        "[UNK]",
        "[CLS]",
        "[SEP]",
        "[PAD]",
        "[MASK]",
        "want",
        "##want",
        "##ed",
        "wa",
        "un",
        "runn",
        "##ing",
        ",",
        "low",
        "lowest",
    ]
    UpperCamelCase__			    =    os.path.join(self.tmpdirname,       VOCAB_FILES_NAMES["vocab_file"]       )
    with open(self.vocab_file,       "w",       encoding="utf-8"       ) as vocab_writer:
       vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )

 def 		lowercase_    (			self			:						str,       a_			:						Dict       ):





    """simple docstring"""



    UpperCamelCase__			    =    "UNwant\u00E9d,running"
    UpperCamelCase__			    =    "unwanted, running"
    return input_text, output_text

 def 		lowercase_    (			self			:						int       ):





    """simple docstring"""



    UpperCamelCase__			    =    self.tokenizer_class(self.vocab_file       )

    UpperCamelCase__			    =    tokenizer.tokenize("UNwant\u00E9d,running"       )
    self.assertListEqual(a_,       ["un", "##want", "##ed", ",", "runn", "##ing"]       )
    self.assertListEqual(tokenizer.convert_tokens_to_ids(a_       ),       [9, 6, 7, 12, 10, 11]       )

 def 		lowercase_    (			self			:						Any       ):





    """simple docstring"""



    if not self.test_rust_tokenizer:
       return

    UpperCamelCase__			    =    self.get_tokenizer()
    UpperCamelCase__			    =    self.get_rust_tokenizer()

    UpperCamelCase__			    =    "UNwant\u00E9d,running"

    UpperCamelCase__			    =    tokenizer.tokenize(a_       )
    UpperCamelCase__			    =    rust_tokenizer.tokenize(a_       )
    self.assertListEqual(a_,       a_       )

    UpperCamelCase__			    =    tokenizer.encode(a_,       add_special_tokens=a_       )
    UpperCamelCase__			    =    rust_tokenizer.encode(a_,       add_special_tokens=a_       )
    self.assertListEqual(a_,       a_       )

    UpperCamelCase__			    =    self.get_rust_tokenizer()
    UpperCamelCase__			    =    tokenizer.encode(a_       )
    UpperCamelCase__			    =    rust_tokenizer.encode(a_       )
    self.assertListEqual(a_,       a_       )

    # With lower casing
    UpperCamelCase__			    =    self.get_tokenizer(do_lower_case=a_       )
    UpperCamelCase__			    =    self.get_rust_tokenizer(do_lower_case=a_       )

    UpperCamelCase__			    =    "UNwant\u00E9d,running"

    UpperCamelCase__			    =    tokenizer.tokenize(a_       )
    UpperCamelCase__			    =    rust_tokenizer.tokenize(a_       )
    self.assertListEqual(a_,       a_       )

    UpperCamelCase__			    =    tokenizer.encode(a_,       add_special_tokens=a_       )
    UpperCamelCase__			    =    rust_tokenizer.encode(a_,       add_special_tokens=a_       )
    self.assertListEqual(a_,       a_       )

    UpperCamelCase__			    =    self.get_rust_tokenizer()
    UpperCamelCase__			    =    tokenizer.encode(a_       )
    UpperCamelCase__			    =    rust_tokenizer.encode(a_       )
    self.assertListEqual(a_,       a_       )

 def 		lowercase_    (			self			:						Any       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer()

    self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       ),       ["ah", "\u535A", "\u63A8", "zz"]       )

 def 		lowercase_    (			self			:						Dict       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       ),       ["hello", "!", "how", "are", "you", "?"]       )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       ),       ["hello"]       )

 def 		lowercase_    (			self			:						Any       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_,       strip_accents=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       ),       ["hällo", "!", "how", "are", "you", "?"]       )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       ),       ["h\u00E9llo"]       )

 def 		lowercase_    (			self			:						Union[str, Any]       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_,       strip_accents=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       ),       ["hallo", "!", "how", "are", "you", "?"]       )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       ),       ["hello"]       )

 def 		lowercase_    (			self			:						int       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       ),       ["hallo", "!", "how", "are", "you", "?"]       )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       ),       ["hello"]       )

 def 		lowercase_    (			self			:						List[str]       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       ),       ["HeLLo", "!", "how", "Are", "yoU", "?"]       )

 def 		lowercase_    (			self			:						Optional[Any]       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_,       strip_accents=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       ),       ["HäLLo", "!", "how", "Are", "yoU", "?"]       )

 def 		lowercase_    (			self			:						Dict       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_,       strip_accents=a_       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       ),       ["HaLLo", "!", "how", "Are", "yoU", "?"]       )

 def 		lowercase_    (			self			:						Optional[int]       ):





    """simple docstring"""



    UpperCamelCase__			    =    BasicTokenizer(do_lower_case=a_,       never_split=["[UNK]"]       )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       ),       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )

 def 		lowercase_    (			self			:						Any       ):





    """simple docstring"""



    UpperCamelCase__			    =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

    UpperCamelCase__			    =    {}
    for i, token in enumerate(a_       ):
       UpperCamelCase__			    =    i
    UpperCamelCase__			    =    WordpieceTokenizer(vocab=a_,       unk_token="[UNK]"       )

    self.assertListEqual(tokenizer.tokenize(""       ),       []       )

    self.assertListEqual(tokenizer.tokenize("unwanted running"       ),       ["un", "##want", "##ed", "runn", "##ing"]       )

    self.assertListEqual(tokenizer.tokenize("unwantedX running"       ),       ["[UNK]", "runn", "##ing"]       )

 def 		lowercase_    (			self			:						Optional[Any]       ):





    """simple docstring"""



    self.assertTrue(_is_whitespace(" "       )       )
    self.assertTrue(_is_whitespace("\t"       )       )
    self.assertTrue(_is_whitespace("\r"       )       )
    self.assertTrue(_is_whitespace("\n"       )       )
    self.assertTrue(_is_whitespace("\u00A0"       )       )

    self.assertFalse(_is_whitespace("A"       )       )
    self.assertFalse(_is_whitespace("-"       )       )

 def 		lowercase_    (			self			:						str       ):





    """simple docstring"""



    self.assertTrue(_is_control("\u0005"       )       )

    self.assertFalse(_is_control("A"       )       )
    self.assertFalse(_is_control(" "       )       )
    self.assertFalse(_is_control("\t"       )       )
    self.assertFalse(_is_control("\r"       )       )

 def 		lowercase_    (			self			:						Dict       ):





    """simple docstring"""



    self.assertTrue(_is_punctuation("-"       )       )
    self.assertTrue(_is_punctuation("$"       )       )
    self.assertTrue(_is_punctuation("`"       )       )
    self.assertTrue(_is_punctuation("."       )       )

    self.assertFalse(_is_punctuation("A"       )       )
    self.assertFalse(_is_punctuation(" "       )       )

 def 		lowercase_    (			self			:						List[Any]       ):





    """simple docstring"""



    UpperCamelCase__			    =    self.get_tokenizer()
    UpperCamelCase__			    =    self.get_rust_tokenizer()

    # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
    self.assertListEqual([tokenizer.tokenize(a_       ) for t in ["Test", "\xad", "test"]],       [["[UNK]"], [], ["[UNK]"]]       )

    self.assertListEqual(
        [rust_tokenizer.tokenize(a_       ) for t in ["Test", "\xad", "test"]],       [["[UNK]"], [], ["[UNK]"]]       )

 @slow
 def 		lowercase_    (			self			:						Dict       ):





    """simple docstring"""



    UpperCamelCase__			    =    self.tokenizer_class.from_pretrained("google/electra-base-discriminator"       )

    UpperCamelCase__			    =    tokenizer.encode("sequence builders",       add_special_tokens=a_       )
    UpperCamelCase__			    =    tokenizer.encode("multi-sequence build",       add_special_tokens=a_       )

    UpperCamelCase__			    =    tokenizer.build_inputs_with_special_tokens(a_       )
    UpperCamelCase__			    =    tokenizer.build_inputs_with_special_tokens(a_,       a_       )

    assert encoded_sentence == [101] + text + [102]
    assert encoded_pair == [101] + text + [102] + text_a + [102]

 def 		lowercase_    (			self			:						List[str]       ):





    """simple docstring"""



    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
          UpperCamelCase__			    =    self.rust_tokenizer_class.from_pretrained(a_,       **a_       )

          UpperCamelCase__			    =    f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
          UpperCamelCase__			    =    tokenizer_r.encode_plus(
              a_,       return_attention_mask=a_,       return_token_type_ids=a_,       return_offsets_mapping=a_,       add_special_tokens=a_,       )

          UpperCamelCase__			    =    tokenizer_r.do_lower_case if hasattr(a_,       "do_lower_case"       ) else False
          UpperCamelCase__			    =    (
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), "A"),
                  ((1, 2), ","),
                  ((3, 5), "na"),
                  ((5, 6), "##ï"),
                  ((6, 8), "##ve"),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), "Allen"),
                  ((21, 23), "##NL"),
                  ((23, 24), "##P"),
                  ((25, 33), "sentence"),
                  ((33, 34), "."),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), "a"),
                  ((1, 2), ","),
                  ((3, 8), "naive"),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), "allen"),
                  ((21, 23), "##nl"),
                  ((23, 24), "##p"),
                  ((25, 33), "sentence"),
                  ((33, 34), "."),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results],       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
          self.assertEqual([e[0] for e in expected_results],       tokens["offset_mapping"]       )

 def 		lowercase_    (			self			:						List[str]       ):





    """simple docstring"""



    UpperCamelCase__			    =    ["的", "人", "有"]
    UpperCamelCase__			    =    "".join(a_       )
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
          UpperCamelCase__			    =    True
          UpperCamelCase__			    =    self.tokenizer_class.from_pretrained(a_,       **a_       )
          UpperCamelCase__			    =    self.rust_tokenizer_class.from_pretrained(a_,       **a_       )

          UpperCamelCase__			    =    tokenizer_p.encode(a_,       add_special_tokens=a_       )
          UpperCamelCase__			    =    tokenizer_r.encode(a_,       add_special_tokens=a_       )

          UpperCamelCase__			    =    tokenizer_r.convert_ids_to_tokens(a_       )
          UpperCamelCase__			    =    tokenizer_p.convert_ids_to_tokens(a_       )

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(a_,       a_       )
          self.assertListEqual(a_,       a_       )

          UpperCamelCase__			    =    False
          UpperCamelCase__			    =    self.rust_tokenizer_class.from_pretrained(a_,       **a_       )
          UpperCamelCase__			    =    self.tokenizer_class.from_pretrained(a_,       **a_       )

          UpperCamelCase__			    =    tokenizer_r.encode(a_,       add_special_tokens=a_       )
          UpperCamelCase__			    =    tokenizer_p.encode(a_,       add_special_tokens=a_       )

          UpperCamelCase__			    =    tokenizer_r.convert_ids_to_tokens(a_       )
          UpperCamelCase__			    =    tokenizer_p.convert_ids_to_tokens(a_       )

          # it is expected that only the first Chinese character is not preceded by "##".
          UpperCamelCase__			    =    [
              f'##{token}' if idx != 0 else token for idx, token in enumerate(a_       )
          ]
          self.assertListEqual(a_,       a_       )
          self.assertListEqual(a_,       a_       )



'''simple docstring'''

import argparse

import torch

from transformers import ChineseCLIPConfig, ChineseCLIPModel



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					int     ,		_UpperCamelCase   :					List[str]     ,		_UpperCamelCase   :					List[str] )		->       List[str]:





   '''simple docstring'''







   UpperCamelCase__    ,		UpperCamelCase__    ,		UpperCamelCase__			    =    pt_weights[F'{prefix}.in_proj_weight'].chunk(3     ,		dim=0 )
   UpperCamelCase__    ,		UpperCamelCase__    ,		UpperCamelCase__			    =    pt_weights[F'{prefix}.in_proj_bias'].chunk(3     ,		dim=0 )

   UpperCamelCase__			    =    pt_weights[F'{prefix}.out_proj.weight']
   UpperCamelCase__			    =    pt_weights[F'{prefix}.out_proj.bias']

   UpperCamelCase__			    =    q_proj
   UpperCamelCase__			    =    q_proj_bias

   UpperCamelCase__			    =    k_proj
   UpperCamelCase__			    =    k_proj_bias

   UpperCamelCase__			    =    v_proj
   UpperCamelCase__			    =    v_proj_bias

   UpperCamelCase__			    =    out_proj_weights
   UpperCamelCase__			    =    out_proj_bias



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					Union[str, Any]     ,		_UpperCamelCase   :					List[str]     ,		_UpperCamelCase   :					Dict )		->       Tuple:





   '''simple docstring'''







   copy_linear(hf_mlp.fca     ,		_UpperCamelCase     ,		F'{prefix}.c_fc' )
   copy_linear(hf_mlp.fca     ,		_UpperCamelCase     ,		F'{prefix}.c_proj' )



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					Tuple     ,		_UpperCamelCase   :					str     ,		_UpperCamelCase   :					Optional[int] )		->       Any:





   '''simple docstring'''







   UpperCamelCase__			    =    pt_weights[F'{prefix}.weight'].data
   UpperCamelCase__			    =    pt_weights[F'{prefix}.bias'].data



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					int     ,		_UpperCamelCase   :					List[Any]     ,		_UpperCamelCase   :					Any )		->       Union[str, Any]:





   '''simple docstring'''







   copy_linear(hf_layer.layer_norma     ,		_UpperCamelCase     ,		F'{prefix}.ln_1' )
   copy_linear(hf_layer.layer_norma     ,		_UpperCamelCase     ,		F'{prefix}.ln_2' )

   # copy MLP
   copy_mlp(hf_layer.mlp     ,		_UpperCamelCase     ,		F'{prefix}.mlp' )

   # copy attn
   copy_attn_layer(hf_layer.self_attn     ,		_UpperCamelCase     ,		F'{prefix}.attn' )



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					Tuple     ,		_UpperCamelCase   :					Optional[int]     ,		_UpperCamelCase   :					Optional[int] )		->       List[Any]:





   '''simple docstring'''







   for layer_id, hf_layer in enumerate(_UpperCamelCase ):
      copy_layer(_UpperCamelCase     ,		_UpperCamelCase     ,		F'{prefix}.{layer_id}' )



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					Tuple     ,		_UpperCamelCase   :					Dict )		->       Tuple:





   '''simple docstring'''







   UpperCamelCase__			    =    pt_weights["text_projection"].data.T

   # copy text encoder
   for name, param in hf_model.text_model.named_parameters():
      UpperCamelCase__			    =    pt_weights[F'bert.{name}'].data



def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					List[str]     ,		_UpperCamelCase   :					Dict )		->       List[Any]:





   '''simple docstring'''







   UpperCamelCase__			    =    pt_weights["visual.proj"].data.T

   # copy layer norms
   copy_linear(hf_model.vision_model.pre_layrnorm     ,		_UpperCamelCase     ,		"visual.ln_pre" )
   copy_linear(hf_model.vision_model.post_layernorm     ,		_UpperCamelCase     ,		"visual.ln_post" )

   # copy embeddings
   UpperCamelCase__			    =    pt_weights["visual.conv1.weight"].data
   UpperCamelCase__			    =    pt_weights["visual.class_embedding"].data
   UpperCamelCase__			    =    pt_weights["visual.positional_embedding"].data

   # copy encoder
   copy_layers(hf_model.vision_model.encoder.layers     ,		_UpperCamelCase     ,		"visual.transformer.resblocks" )



@torch.no_grad()
def 							SCREAMING_SNAKE_CASE__(		_UpperCamelCase   :					int     ,		_UpperCamelCase   :					Union[str, Any]     ,		_UpperCamelCase   :					Any=None )		->       List[str]:





   '''simple docstring'''







   assert config_path is not None, "Please specify the ChineseCLIP model config of the corresponding model size."
   UpperCamelCase__			    =    ChineseCLIPConfig.from_pretrained(_UpperCamelCase )

   UpperCamelCase__			    =    ChineseCLIPModel(_UpperCamelCase ).eval()

   UpperCamelCase__			    =    torch.load(_UpperCamelCase     ,		map_location="cpu" )["state_dict"]
   UpperCamelCase__			    =    {(name[7:] if name.startswith("module." ) else name): value for name, value in pt_weights.items()}

   copy_text_model_and_projection(_UpperCamelCase     ,		_UpperCamelCase )
   copy_vision_model_and_projection(_UpperCamelCase     ,		_UpperCamelCase )
   UpperCamelCase__			    =    pt_weights["logit_scale"].data

   hf_model.save_pretrained(_UpperCamelCase )


if __name__ == "__main__":
       __lowercase:		Tuple    						=  argparse.ArgumentParser()
       parser.add_argument(
           "--pytorch_dump_folder_path",
           default=None,
           type=str,
           help="Path to the output folder storing converted hf PyTorch model.",
       )
       parser.add_argument(
           "--checkpoint_path", default=None, type=str, help="Path to original github format ChineseCLIP checkpoint."
       )
       parser.add_argument(
           "--config_path", default=None, required=True, type=str, help="Path to hf config.json of model to convert."
       )
       __lowercase:		Optional[int]    						=  parser.parse_args()

       convert_chinese_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)
       print("The conversion is finished!")