'''simple docstring'''


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


snake_case__	:     Any      =    logging.getLogger(__name__)


def    _lowerCamelCase       ( ):



     """simple docstring"""





     UpperCAmelCase_				:  Optional[int]  = argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'  )
     parser.add_argument(
         '--dataset_name'      ,	type=lowerCamelCase_      ,	default='wikitext'      ,	help='Name of the training. Explore datasets at: hf.co/datasets.'      ,	)
     parser.add_argument(
         '--dataset_config'      ,	type=lowerCamelCase_      ,	default='wikitext-103-raw-v1'      ,	help='Configuration name of the dataset.'  )
     parser.add_argument(
         '--batch_size'      ,	type=lowerCamelCase_      ,	default=1000      ,	help='Batch size during training.'      ,	)
     parser.add_argument(
         '--vocab_size'      ,	type=lowerCamelCase_      ,	default=10048      ,	help='Size of the desired vocabulary.'      ,	)
     parser.add_argument(
         '--limit'      ,	default=lowerCamelCase_      ,	type=lowerCamelCase_      ,	help='Limit the number of shards (used for debugging).'      ,	)
     parser.add_argument(
         '--export_to_hub'      ,	action='store_true'      ,	)

     UpperCAmelCase_				:  Tuple  = parser.parse_args()
     return args


def    _lowerCamelCase       ( lowerCamelCase_   :				str  ):



     """simple docstring"""





     UpperCAmelCase_				:  Tuple  = datasets.load_dataset(args.dataset_name      ,	args.dataset_config      ,	split='train'  )

     if args.limit is not None:
          UpperCAmelCase_				:  Dict  = min(len(lowerCamelCase_  )      ,	args.limit  )
          UpperCAmelCase_				:  Optional[Any]  = dataset.select(range(lowerCamelCase_  )  )
          logger.info(F'''Limiting the dataset to {args.limit} entries.'''  )

     def batch_iterator():
          for i in range(0      ,	len(lowerCamelCase_  )      ,	args.batch_size  ):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     UpperCAmelCase_				:  Optional[int]  = Tokenizer(Unigram()  )
     UpperCAmelCase_				:  Optional[int]  = normalizers.Sequence([normalizers.Replace('``'      ,	'"'  ), normalizers.Replace('\'\''      ,	'"'  )]  )
     UpperCAmelCase_				:  Union[str, Any]  = pre_tokenizers.Metaspace()

     # Prepare the trainer.
     UpperCAmelCase_				:  str  = UnigramTrainer(
         unk_token='<unk>'      ,	special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']      ,	vocab_size=args.vocab_size      ,	)

     logger.info('Training the tokenizer.'  )
     tokenizer.train_from_iterator(batch_iterator()      ,	trainer=lowerCamelCase_  )
     logger.info('Tokenizer training complete!'  )

     UpperCAmelCase_				:  Optional[Any]  = tokenizer.token_to_id('[CLS]'  )
     UpperCAmelCase_				:  List[str]  = tokenizer.token_to_id('[SEP]'  )
     UpperCAmelCase_				:  List[str]  = processors.TemplateProcessing(
         single='[CLS]:0 $A:0 [SEP]:0'      ,	pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'      ,	special_tokens=[
             ('[CLS]', cls_token_id),
             ('[SEP]', sep_token_id),
         ]      ,	)
     UpperCAmelCase_				:  List[str]  = decoders.Metaspace()

     if args.export_to_hub:
          logger.info('Exporting the trained tokenzier to Hub.'  )
          UpperCAmelCase_				:  Tuple  = AlbertTokenizerFast(tokenizer_object=lowerCamelCase_  )
          new_tokenizer.push_to_hub('unigram-tokenizer-dataset'  )


if __name__ == "__main__":
 snake_case__	:     Optional[int]      =    parse_args()
 main(args)










import inspect
import unittest

from transformers import ResNetConfig, is_flax_available
from transformers.testing_utils import require_flax, slow
from transformers.utils import cached_property, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
    import jax
    import jax.numpy as jnp

    from transformers.models.resnet.modeling_flax_resnet import FlaxResNetForImageClassification, FlaxResNetModel

if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor







class      _lowercase					(				unittest.TestCase):


       """simple docstring"""




       def __init__(  self				:     Dict   , __lowerCamelCase				:     int   , __lowerCamelCase				:     List[Any]=3   , __lowerCamelCase				:     Dict=32   , __lowerCamelCase				:     Optional[Any]=3   , __lowerCamelCase				:     List[str]=10   , __lowerCamelCase				:     List[str]=[10, 20, 30, 40]   , __lowerCamelCase				:     List[Any]=[1, 1, 2, 1]   , __lowerCamelCase				:     Any=True   , __lowerCamelCase				:     Optional[int]=True   , __lowerCamelCase				:     int="relu"   , __lowerCamelCase				:     List[str]=3   , __lowerCamelCase				:     Any=None   , ):


             '''simple docstring'''



             lowerCamelCase__						:							int       =    parent
             lowerCamelCase__						:							Optional[int]       =    batch_size
             lowerCamelCase__						:							List[Any]       =    image_size
             lowerCamelCase__						:							Union[str, Any]       =    num_channels
             lowerCamelCase__						:							Any       =    embeddings_size
             lowerCamelCase__						:							Tuple       =    hidden_sizes
             lowerCamelCase__						:							Tuple       =    depths
             lowerCamelCase__						:							List[str]       =    is_training
             lowerCamelCase__						:							Optional[Any]       =    use_labels
             lowerCamelCase__						:							str       =    hidden_act
             lowerCamelCase__						:							Any       =    num_labels
             lowerCamelCase__						:							List[str]       =    scope
             lowerCamelCase__						:							Any       =    len(__lowerCamelCase			)





       def 		lowerCAmelCase	(  self				:     List[str]			):


             '''simple docstring'''



             lowerCamelCase__						:							Any       =    floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

             lowerCamelCase__						:							Any       =    self.get_config()

             return config, pixel_values





       def 		lowerCAmelCase	(  self				:     Optional[int]			):


             '''simple docstring'''



             return ResNetConfig(
                 num_channels=self.num_channels   , embeddings_size=self.embeddings_size   , hidden_sizes=self.hidden_sizes   , depths=self.depths   , hidden_act=self.hidden_act   , num_labels=self.num_labels   , image_size=self.image_size   , )





       def 		lowerCAmelCase	(  self				:     str   , __lowerCamelCase				:     List[Any]   , __lowerCamelCase				:     Tuple			):


             '''simple docstring'''



             lowerCamelCase__						:							Any       =    FlaxResNetModel(config=__lowerCamelCase			)
             lowerCamelCase__						:							str       =    model(__lowerCamelCase			)

             # Output shape (b, c, h, w)
             self.parent.assertEqual(
                 result.last_hidden_state.shape   , (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)   , )





       def 		lowerCAmelCase	(  self				:     int   , __lowerCamelCase				:     Optional[Any]   , __lowerCamelCase				:     List[str]			):


             '''simple docstring'''



             lowerCamelCase__						:							Optional[Any]       =    self.num_labels
             lowerCamelCase__						:							int       =    FlaxResNetForImageClassification(config=__lowerCamelCase			)
             lowerCamelCase__						:							Optional[int]       =    model(__lowerCamelCase			)
             self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.num_labels)			)





       def 		lowerCAmelCase	(  self				:     Dict			):


             '''simple docstring'''



             lowerCamelCase__						:							Tuple       =    self.prepare_config_and_inputs()
             lowerCamelCase__       ,  lowerCamelCase__						:							Tuple       =    config_and_inputs
             lowerCamelCase__						:							Optional[int]       =    {"pixel_values": pixel_values}
             return config, inputs_dict







@require_flax
class      _lowercase					(				lowercase__				,    unittest.TestCase):


       """simple docstring"""




       A__   		=     (FlaxResNetModel, FlaxResNetForImageClassification) if is_flax_available() else ()

       A__   		=     False
       A__   		=     False
       A__   		=     False





       def 		lowerCAmelCase	(  self				:     List[str]			):


             '''simple docstring'''



             lowerCamelCase__						:							List[str]       =    FlaxResNetModelTester(self			)
             lowerCamelCase__						:							Dict       =    ConfigTester(self   , config_class=__lowerCamelCase   , has_text_modality=__lowerCamelCase			)





       def 		lowerCAmelCase	(  self				:     Optional[Any]			):


             '''simple docstring'''



             self.create_and_test_config_common_properties()
             self.config_tester.create_and_test_config_to_json_string()
             self.config_tester.create_and_test_config_to_json_file()
             self.config_tester.create_and_test_config_from_and_save_pretrained()
             self.config_tester.create_and_test_config_with_num_labels()
             self.config_tester.check_config_can_be_init_without_params()
             self.config_tester.check_config_arguments_init()





       def 		lowerCAmelCase	(  self				:     Dict			):


             '''simple docstring'''



             return





       def 		lowerCAmelCase	(  self				:     Dict			):


             '''simple docstring'''



             lowerCamelCase__						:							str       =    self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*__lowerCamelCase			)





       def 		lowerCAmelCase	(  self				:     List[Any]			):


             '''simple docstring'''



             lowerCamelCase__						:							str       =    self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_image_classification(*__lowerCamelCase			)





       @unittest.skip(reason="ResNet does not use inputs_embeds"			)
       def 		lowerCAmelCase	(  self				:     Dict			):


             '''simple docstring'''



             pass





       @unittest.skip(reason="ResNet does not support input and output embeddings"			)
       def 		lowerCAmelCase	(  self				:     int			):


             '''simple docstring'''



             pass





       def 		lowerCAmelCase	(  self				:     Any			):


             '''simple docstring'''



             lowerCamelCase__       ,  lowerCamelCase__						:							Tuple       =    self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                   lowerCamelCase__						:							Tuple       =    model_class(__lowerCamelCase			)
                   lowerCamelCase__						:							Tuple       =    inspect.signature(model.__call__			)
                   # signature.parameters is an OrderedDict => so arg_names order is deterministic
                   lowerCamelCase__						:							Optional[Any]       =    [*signature.parameters.keys()]

                   lowerCamelCase__						:							List[Any]       =    ["pixel_values"]
                   self.assertListEqual(arg_names[:1]   , __lowerCamelCase			)





       def 		lowerCAmelCase	(  self				:     Any			):


             '''simple docstring'''



             def check_hidden_states_output(__lowerCamelCase				:     List[str]   , __lowerCamelCase				:     Optional[int]   , __lowerCamelCase				:     str			):
                   lowerCamelCase__						:							Any       =    model_class(__lowerCamelCase			)

                   lowerCamelCase__						:							Dict       =    model(**self._prepare_for_class(__lowerCamelCase   , __lowerCamelCase			)			)

                   lowerCamelCase__						:							str       =    outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                   lowerCamelCase__						:							Tuple       =    self.model_tester.num_stages
                   self.assertEqual(len(__lowerCamelCase			)   , expected_num_stages + 1			)





       @unittest.skip(reason="ResNet does not use feedforward chunking"			)
       def 		lowerCAmelCase	(  self				:     Any			):


             '''simple docstring'''



             pass





       def 		lowerCAmelCase	(  self				:     int			):


             '''simple docstring'''



             lowerCamelCase__       ,  lowerCamelCase__						:							Dict       =    self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                   with self.subTest(model_class.__name__			):
                         lowerCamelCase__						:							Any       =    self._prepare_for_class(__lowerCamelCase   , __lowerCamelCase			)
                         lowerCamelCase__						:							Optional[int]       =    model_class(__lowerCamelCase			)

                         @jax.jit
                         def model_jitted(__lowerCamelCase				:     Optional[Any]   , **__lowerCamelCase				:     List[Any]			):
                               return model(pixel_values=__lowerCamelCase   , **__lowerCamelCase			)

                         with self.subTest("JIT Enabled"			):
                               lowerCamelCase__						:							Any       =    model_jitted(**__lowerCamelCase			).to_tuple()

                         with self.subTest("JIT Disabled"			):
                               with jax.disable_jit():
                                     lowerCamelCase__						:							Union[str, Any]       =    model_jitted(**__lowerCamelCase			).to_tuple()

                         self.assertEqual(len(__lowerCamelCase			)   , len(__lowerCamelCase			)			)
                         for jitted_output, output in zip(__lowerCamelCase   , __lowerCamelCase			):
                               self.assertEqual(jitted_output.shape   , output.shape			)



def  lowercase_						(						):

      """simple docstring"""




      lowerCamelCase__						:							Optional[Any]       =    Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
      return image







@require_flax
class      _lowercase					(				unittest.TestCase):


       """simple docstring"""




       @cached_property
       def 		lowerCAmelCase	(  self				:     Tuple			):


             '''simple docstring'''



             return AutoImageProcessor.from_pretrained("microsoft/resnet-50"			) if is_vision_available() else None





       @slow
       def 		lowerCAmelCase	(  self				:     List[str]			):


             '''simple docstring'''



             lowerCamelCase__						:							List[Any]       =    FlaxResNetForImageClassification.from_pretrained("microsoft/resnet-50"			)

             lowerCamelCase__						:							List[str]       =    self.default_image_processor
             lowerCamelCase__						:							Optional[Any]       =    prepare_img()
             lowerCamelCase__						:							str       =    image_processor(images=__lowerCamelCase   , return_tensors="np"			)

             lowerCamelCase__						:							str       =    model(**__lowerCamelCase			)

             # verify the logits
             lowerCamelCase__						:							int       =    (1, 1000)
             self.assertEqual(outputs.logits.shape   , __lowerCamelCase			)

             lowerCamelCase__						:							Optional[int]       =    jnp.array([-1_1.1_0_6_9, -9.7_8_7_7, -8.3_7_7_7]			)

             self.assertTrue(jnp.allclose(outputs.logits[0, :3]   , __lowerCamelCase   , atol=1E-4			)			)


