


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class        UpperCAmelCase__			(   A_  ,    unittest.TestCase						):



    """simple docstring"""






    UpperCAmelCase__		:						Union[str, Any]    =    ElectraTokenizer
    UpperCAmelCase__		:						Dict    =    ElectraTokenizerFast
    UpperCAmelCase__		:						Dict    =    True
    UpperCAmelCase__		:						Optional[Any]    =    True
    UpperCAmelCase__		:						List[Any]    =    filter_non_english






    def _a   (     self       )    ->			int:
         super().setUp()

         __UpperCamelCase       				=[
             '[UNK]',
             '[CLS]',
             '[SEP]',
             '[PAD]',
             '[MASK]',
             'want',
             '##want',
             '##ed',
             'wa',
             'un',
             'runn',
             '##ing',
             ',',
             'low',
             'lowest',
         ]
         __UpperCamelCase       				=os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES['vocab_file']       )
         with open(self.vocab_file   ,  'w'   ,  encoding='utf-8'       ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )






    def _a   (     self   ,  A_       )    ->			Any:
         __UpperCamelCase       				='UNwant\u00E9d,running'
         __UpperCamelCase       				='unwanted, running'
         return input_text, output_text






    def _a   (     self       )    ->			List[Any]:
         __UpperCamelCase       				=self.tokenizer_class(self.vocab_file       )

         __UpperCamelCase       				=tokenizer.tokenize('UNwant\u00E9d,running'       )
         self.assertListEqual(A_   ,  ['un', '##want', '##ed', ',', 'runn', '##ing']       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A_       )   ,  [9, 6, 7, 12, 10, 11]       )






    def _a   (     self       )    ->			List[Any]:
         if not self.test_rust_tokenizer:
              return

         __UpperCamelCase       				=self.get_tokenizer()
         __UpperCamelCase       				=self.get_rust_tokenizer()

         __UpperCamelCase       				='UNwant\u00E9d,running'

         __UpperCamelCase       				=tokenizer.tokenize(A_       )
         __UpperCamelCase       				=rust_tokenizer.tokenize(A_       )
         self.assertListEqual(A_   ,  A_       )

         __UpperCamelCase       				=tokenizer.encode(A_   ,  add_special_tokens=A_       )
         __UpperCamelCase       				=rust_tokenizer.encode(A_   ,  add_special_tokens=A_       )
         self.assertListEqual(A_   ,  A_       )

         __UpperCamelCase       				=self.get_rust_tokenizer()
         __UpperCamelCase       				=tokenizer.encode(A_       )
         __UpperCamelCase       				=rust_tokenizer.encode(A_       )
         self.assertListEqual(A_   ,  A_       )

         # With lower casing
         __UpperCamelCase       				=self.get_tokenizer(do_lower_case=A_       )
         __UpperCamelCase       				=self.get_rust_tokenizer(do_lower_case=A_       )

         __UpperCamelCase       				='UNwant\u00E9d,running'

         __UpperCamelCase       				=tokenizer.tokenize(A_       )
         __UpperCamelCase       				=rust_tokenizer.tokenize(A_       )
         self.assertListEqual(A_   ,  A_       )

         __UpperCamelCase       				=tokenizer.encode(A_   ,  add_special_tokens=A_       )
         __UpperCamelCase       				=rust_tokenizer.encode(A_   ,  add_special_tokens=A_       )
         self.assertListEqual(A_   ,  A_       )

         __UpperCamelCase       				=self.get_rust_tokenizer()
         __UpperCamelCase       				=tokenizer.encode(A_       )
         __UpperCamelCase       				=rust_tokenizer.encode(A_       )
         self.assertListEqual(A_   ,  A_       )






    def _a   (     self       )    ->			int:
         __UpperCamelCase       				=BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'       )   ,  ['ah', '\u535A', '\u63A8', 'zz']       )






    def _a   (     self       )    ->			List[str]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )   ,  ['hello', '!', 'how', 'are', 'you', '?']       )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )   ,  ['hello']       )






    def _a   (     self       )    ->			List[Any]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_   ,  strip_accents=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )   ,  ['hällo', '!', 'how', 'are', 'you', '?']       )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )   ,  ['h\u00E9llo']       )






    def _a   (     self       )    ->			Optional[int]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_   ,  strip_accents=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )   ,  ['hallo', '!', 'how', 'are', 'you', '?']       )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )   ,  ['hello']       )






    def _a   (     self       )    ->			Union[str, Any]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )   ,  ['hallo', '!', 'how', 'are', 'you', '?']       )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )   ,  ['hello']       )






    def _a   (     self       )    ->			List[Any]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )   ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']       )






    def _a   (     self       )    ->			int:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_   ,  strip_accents=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )   ,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']       )






    def _a   (     self       )    ->			Any:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_   ,  strip_accents=A_       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )   ,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']       )






    def _a   (     self       )    ->			List[str]:
         __UpperCamelCase       				=BasicTokenizer(do_lower_case=A_   ,  never_split=['[UNK]']       )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'       )   ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']       )






    def _a   (     self       )    ->			Optional[Any]:
         __UpperCamelCase       				=['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

         __UpperCamelCase       				={}
         for i, token in enumerate(A_       ):
              __UpperCamelCase       				=i
         __UpperCamelCase       				=WordpieceTokenizer(vocab=A_   ,  unk_token='[UNK]'       )

         self.assertListEqual(tokenizer.tokenize(''       )   ,  []       )

         self.assertListEqual(tokenizer.tokenize('unwanted running'       )   ,  ['un', '##want', '##ed', 'runn', '##ing']       )

         self.assertListEqual(tokenizer.tokenize('unwantedX running'       )   ,  ['[UNK]', 'runn', '##ing']       )






    def _a   (     self       )    ->			Optional[Any]:
         self.assertTrue(_is_whitespace(' '       )       )
         self.assertTrue(_is_whitespace('\t'       )       )
         self.assertTrue(_is_whitespace('\r'       )       )
         self.assertTrue(_is_whitespace('\n'       )       )
         self.assertTrue(_is_whitespace('\u00A0'       )       )

         self.assertFalse(_is_whitespace('A'       )       )
         self.assertFalse(_is_whitespace('-'       )       )






    def _a   (     self       )    ->			Optional[int]:
         self.assertTrue(_is_control('\u0005'       )       )

         self.assertFalse(_is_control('A'       )       )
         self.assertFalse(_is_control(' '       )       )
         self.assertFalse(_is_control('\t'       )       )
         self.assertFalse(_is_control('\r'       )       )






    def _a   (     self       )    ->			Dict:
         self.assertTrue(_is_punctuation('-'       )       )
         self.assertTrue(_is_punctuation('$'       )       )
         self.assertTrue(_is_punctuation('`'       )       )
         self.assertTrue(_is_punctuation('.'       )       )

         self.assertFalse(_is_punctuation('A'       )       )
         self.assertFalse(_is_punctuation(' '       )       )






    def _a   (     self       )    ->			Tuple:
         __UpperCamelCase       				=self.get_tokenizer()
         __UpperCamelCase       				=self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(A_       ) for t in ['Test', '\xad', 'test']]   ,  [['[UNK]'], [], ['[UNK]']]       )

         self.assertListEqual(
             [rust_tokenizer.tokenize(A_       ) for t in ['Test', '\xad', 'test']]   ,  [['[UNK]'], [], ['[UNK]']]       )






    @slow
    def _a   (     self       )    ->			int:
         __UpperCamelCase       				=self.tokenizer_class.from_pretrained('google/electra-base-discriminator'       )

         __UpperCamelCase       				=tokenizer.encode('sequence builders'   ,  add_special_tokens=A_       )
         __UpperCamelCase       				=tokenizer.encode('multi-sequence build'   ,  add_special_tokens=A_       )

         __UpperCamelCase       				=tokenizer.build_inputs_with_special_tokens(A_       )
         __UpperCamelCase       				=tokenizer.build_inputs_with_special_tokens(A_   ,  A_       )

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]






    def _a   (     self       )    ->			str:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
                   __UpperCamelCase       				=self.rust_tokenizer_class.from_pretrained(A_   ,  **A_       )

                   __UpperCamelCase       				=f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                   __UpperCamelCase       				=tokenizer_r.encode_plus(
                       A_   ,  return_attention_mask=A_   ,  return_token_type_ids=A_   ,  return_offsets_mapping=A_   ,  add_special_tokens=A_   ,  )

                   __UpperCamelCase       				=tokenizer_r.do_lower_case if hasattr(A_   ,  'do_lower_case'       ) else False
                   __UpperCamelCase       				=(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]   ,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']       )       )
                   self.assertEqual([e[0] for e in expected_results]   ,  tokens['offset_mapping']       )






    def _a   (     self       )    ->			int:
         __UpperCamelCase       				=['的', '人', '有']
         __UpperCamelCase       				=''.join(A_       )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
                   __UpperCamelCase       				=True
                   __UpperCamelCase       				=self.tokenizer_class.from_pretrained(A_   ,  **A_       )
                   __UpperCamelCase       				=self.rust_tokenizer_class.from_pretrained(A_   ,  **A_       )

                   __UpperCamelCase       				=tokenizer_p.encode(A_   ,  add_special_tokens=A_       )
                   __UpperCamelCase       				=tokenizer_r.encode(A_   ,  add_special_tokens=A_       )

                   __UpperCamelCase       				=tokenizer_r.convert_ids_to_tokens(A_       )
                   __UpperCamelCase       				=tokenizer_p.convert_ids_to_tokens(A_       )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(A_   ,  A_       )
                   self.assertListEqual(A_   ,  A_       )

                   __UpperCamelCase       				=False
                   __UpperCamelCase       				=self.rust_tokenizer_class.from_pretrained(A_   ,  **A_       )
                   __UpperCamelCase       				=self.tokenizer_class.from_pretrained(A_   ,  **A_       )

                   __UpperCamelCase       				=tokenizer_r.encode(A_   ,  add_special_tokens=A_       )
                   __UpperCamelCase       				=tokenizer_p.encode(A_   ,  add_special_tokens=A_       )

                   __UpperCamelCase       				=tokenizer_r.convert_ids_to_tokens(A_       )
                   __UpperCamelCase       				=tokenizer_p.convert_ids_to_tokens(A_       )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   __UpperCamelCase       				=[
                       f'##{token}' if idx != 0 else token for idx, token in enumerate(A_       )
                   ]
                   self.assertListEqual(A_   ,  A_       )
                   self.assertListEqual(A_   ,  A_       )









import argparse
import json

from transformers import GPTNeoConfig, GPTNeoForCausalLM, load_tf_weights_in_gpt_neo
from transformers.utils import logging


logging.set_verbosity_info()

def 		_UpperCAmelCase							(     SCREAMING_SNAKE_CASE__     :   List[Any]     ,	SCREAMING_SNAKE_CASE__     :   Union[str, Any]     ,	SCREAMING_SNAKE_CASE__     :   Any							):
     # Initialise PyTorch model
     __UpperCamelCase       				=json.load(open(SCREAMING_SNAKE_CASE__     ,	'r'							)							)
     __UpperCamelCase       				=GPTNeoConfig(
         hidden_size=config_json['n_embd']     ,	num_layers=config_json['n_layer']     ,	num_heads=config_json['n_head']     ,	attention_types=config_json['attention_types']     ,	max_position_embeddings=config_json['n_positions']     ,	resid_dropout=config_json['res_dropout']     ,	embed_dropout=config_json['embed_dropout']     ,	attention_dropout=config_json['attn_dropout']     ,	)
     print(F'Building PyTorch model from configuration: {config}'							)
     __UpperCamelCase       				=GPTNeoForCausalLM(SCREAMING_SNAKE_CASE__							)

     # Load weights from tf checkpoint
     load_tf_weights_in_gpt_neo(SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__     ,	SCREAMING_SNAKE_CASE__							)

     # Save pytorch-model
     print(F'Save PyTorch model to {pytorch_dump_path}'							)
     model.save_pretrained(SCREAMING_SNAKE_CASE__							)


if __name__ == "__main__":
   _A									=		argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
   )
   parser.add_argument(
       '--config_file',
       default=None,
       type=str,
       required=True,
       help=(
           'The config json file corresponding to the pre-trained mesh-tf model. \n'
           'This specifies the model architecture.'
       ),
   )
   parser.add_argument(
       '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   _A									=		parser.parse_args()
   convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)





