





import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__lowerCAmelCase			:str	 =     16
__lowerCAmelCase			:Union[str, Any]	 =     32







def  A					(				UpperCAmelCase					):
					return int(x / 2**20					)






class 							_a:







		def __enter__(     self					)   ->   List[Any]:


							'''simple docstring'''

							gc.collect()
							torch.cuda.empty_cache()
							torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
							_snake_case							:       str												= torch.cuda.memory_allocated()
							_snake_case							:       Optional[Any]												= psutil.Process()

							_snake_case							:       int												= self.cpu_mem_used()
							_snake_case							:       Dict												= True
							_snake_case							:       int												= threading.Thread(target=self.peak_monitor_func					)
							_snake_case							:       Union[str, Any]												= True
							peak_monitor_thread.start()
							return self







		def 							lowercase							(     self					)   ->   int:


							'''simple docstring'''

							return self.process.memory_info().rss







		def 							lowercase							(     self					)   ->   Dict:


							'''simple docstring'''

							_snake_case							:       List[str]												= -1

							while True:
												_snake_case							:       Union[str, Any]												= max(self.cpu_mem_used()			,    self.cpu_peak					)

												# can't sleep or will not catch the peak right (this comment is here on purpose)
												# time.sleep(0.001) # 1msec

												if not self.peak_monitoring:
																	break







		def __exit__(     self			,    *__snake_case					)   ->   List[Any]:


							'''simple docstring'''

							_snake_case							:       List[str]												= False

							gc.collect()
							torch.cuda.empty_cache()
							_snake_case							:       Optional[Any]												= torch.cuda.memory_allocated()
							_snake_case							:       Dict												= torch.cuda.max_memory_allocated()
							_snake_case							:       Any												= bamb(self.end - self.begin					)
							_snake_case							:       List[Any]												= bamb(self.peak - self.begin					)

							_snake_case							:       List[str]												= self.cpu_mem_used()
							_snake_case							:       Tuple												= bamb(self.cpu_end - self.cpu_begin					)
							_snake_case							:       List[Any]												= bamb(self.cpu_peak - self.cpu_begin					)
							# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							__lowerCAmelCase			:List[Any]	 =     mocked_dataloaders  # noqa: F811







def  A					(				UpperCAmelCase	,	UpperCAmelCase					):


					if os.environ.get("TESTING_MOCKED_DATALOADERS"	,	snake_case_					) == "1":
										_snake_case							:       Tuple												= 2

					# New Code #
					# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
					_snake_case							:       List[str]												= FullyShardedDataParallelPlugin(
					    state_dict_config=FullStateDictConfig(offload_to_cpu=snake_case_	,	ranka_only=snake_case_					)	,	optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=snake_case_	,	ranka_only=snake_case_					)	,	)

					# Initialize accelerator
					if args.with_tracking:
										_snake_case							:       Optional[int]												= Accelerator(
										    cpu=args.cpu	,	mixed_precision=args.mixed_precision	,	log_with="wandb"	,	project_dir=args.logging_dir	,	fsdp_plugin=snake_case_	,	)
					else:
										_snake_case							:       Tuple												= Accelerator(fsdp_plugin=snake_case_					)
					accelerator.print(accelerator.distributed_type					)

					if hasattr(args.checkpointing_steps	,	"isdigit"					):
										if args.checkpointing_steps == "epoch":
															_snake_case							:       Any												= args.checkpointing_steps
										elif args.checkpointing_steps.isdigit():
															_snake_case							:       Union[str, Any]												= int(args.checkpointing_steps					)
										else:
															raise ValueError(
															    F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""					)
					else:
										_snake_case							:       List[Any]												= None
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					_snake_case							:       int												= config["lr"]
					_snake_case							:       Optional[int]												= int(config["num_epochs"]					)
					_snake_case							:       Any												= int(config["seed"]					)
					_snake_case							:       Optional[int]												= int(config["batch_size"]					)

					# We need to initialize the trackers we use, and also store our configuration
					if args.with_tracking:
										_snake_case							:       Tuple												= vars(snake_case_					)
										accelerator.init_trackers("fsdp_glue_no_trainer"	,	snake_case_					)

					_snake_case							:       Optional[Any]												= AutoTokenizer.from_pretrained(args.model_name_or_path					)
					_snake_case							:       List[Any]												= load_dataset("glue"	,	"mrpc"					)
					_snake_case							:       str												= evaluate.load("glue"	,	"mrpc"					)

					def tokenize_function(UpperCAmelCase					):
										# max_length=None => use the model max length (it's actually the default)
										_snake_case							:       List[str]												= tokenizer(examples["sentence1"]	,	examples["sentence2"]	,	truncation=snake_case_	,	max_length=snake_case_					)
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										_snake_case							:       Union[str, Any]												= datasets.map(
										    snake_case_	,	batched=snake_case_	,	remove_columns=["idx", "sentence1", "sentence2"]	,	)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					_snake_case							:       Tuple												= tokenized_datasets.rename_column("label"	,	"labels"					)

					# If the batch size is too big we use gradient accumulation
					_snake_case							:       Union[str, Any]												= 1
					if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
										_snake_case							:       Dict												= batch_size // MAX_GPU_BATCH_SIZE
										_snake_case							:       Optional[int]												= MAX_GPU_BATCH_SIZE

					def collate_fn(UpperCAmelCase					):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										_snake_case							:       List[Any]												= 128 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															_snake_case							:       str												= 16
										elif accelerator.mixed_precision != "no":
															_snake_case							:       Dict												= 8
										else:
															_snake_case							:       Union[str, Any]												= None

										return tokenizer.pad(
										    snake_case_	,	padding="longest"	,	max_length=snake_case_	,	pad_to_multiple_of=snake_case_	,	return_tensors="pt"	,	)

					# Instantiate dataloaders.
					_snake_case							:       List[Any]												= DataLoader(
					    tokenized_datasets["train"]	,	shuffle=snake_case_	,	collate_fn=snake_case_	,	batch_size=snake_case_					)
					_snake_case							:       Optional[Any]												= DataLoader(
					    tokenized_datasets["validation"]	,	shuffle=snake_case_	,	collate_fn=snake_case_	,	batch_size=snake_case_					)

					set_seed(snake_case_					)

					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					_snake_case							:       Any												= AutoModelForSequenceClassification.from_pretrained(
					    args.model_name_or_path	,	return_dict=snake_case_	,	low_cpu_mem_usage=snake_case_					)

					# New Code #
					# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
					_snake_case							:       int												= accelerator.prepare(snake_case_					)
					accelerator.print(snake_case_					)

					# Instantiate optimizer
					# New Code #
					# For FSDP feature, at present it doesn't support multiple parameter groups,
					# so we need to create a single parameter group for the whole model
					_snake_case							:       List[Any]												= torch.optim.AdamW(params=model.parameters()	,	lr=snake_case_	,	weight_decay=2e-4					)

					# Instantiate scheduler
					_snake_case							:       int												= get_linear_schedule_with_warmup(
					    optimizer=snake_case_	,	num_warmup_steps=10	,	num_training_steps=(len(snake_case_					) * num_epochs) // gradient_accumulation_steps	,	)

					# New Code #
					# For FSDP feature, prepare everything except the model as we have already prepared the model
					# before creating the optimizer
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					_snake_case							,			_snake_case							,			_snake_case							,			_snake_case							:       Union[str, Any]												= accelerator.prepare(
					    snake_case_	,	snake_case_	,	snake_case_	,	snake_case_					)

					_snake_case							:       Dict												= 0

					# Potentially load in the weights and states from a previous save
					if args.resume_from_checkpoint:
										if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
															accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""					)
															accelerator.load_state(args.resume_from_checkpoint					)
															_snake_case							:       Union[str, Any]												= os.path.basename(args.resume_from_checkpoint					)
										else:
															# Get the most recent checkpoint
															_snake_case							:       int												= [f.name for f in os.scandir(os.getcwd()					) if f.is_dir()]
															dirs.sort(key=os.path.getctime					)
															_snake_case							:       int												= dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
										# Extract `epoch_{i}` or `step_{i}`
										_snake_case							:       Tuple												= os.path.splitext(snake_case_					)[0]

										if "epoch" in training_difference:
															num_epochs -= int(training_difference.replace("epoch_"	,	""					)					)
															_snake_case							:       Union[str, Any]												= None
										else:
															_snake_case							:       Optional[Any]												= int(training_difference.replace("step_"	,	""					)					)
															num_epochs -= resume_step // len(snake_case_					)
															# If resuming by step, we also need to know exactly how far into the DataLoader we went
															_snake_case							:       List[str]												= (num_epochs * len(snake_case_					)) - resume_step

    # Now we train the model
					for epoch in range(snake_case_					):
										# New Code #
										# context manager to track the peak memory usage during the training epoch
										with TorchTracemalloc() as tracemalloc:
															model.train()
															if args.with_tracking:
																				_snake_case							:       int												= 0
															for step, batch in enumerate(snake_case_					):
																				# We need to skip steps until we reach the resumed step
																				if args.resume_from_checkpoint and epoch == 0:
																									if resume_step is not None and step < resume_step:
																														pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device					)
																				_snake_case							:       Optional[Any]												= model(**snake_case_					)
																				_snake_case							:       Optional[int]												= outputs.loss
																				_snake_case							:       Tuple												= loss / gradient_accumulation_steps
																				# We keep track of the loss at each epoch
																				if args.with_tracking:
																									total_loss += loss.detach().float()
																				accelerator.backward(snake_case_					)
																				if step % gradient_accumulation_steps == 0:
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# accelerator.print(lr_scheduler.get_lr())

																				overall_step += 1

																				if isinstance(snake_case_	,	snake_case_					):
																									_snake_case							:       Dict												= F"""step_{overall_step}"""
																									if overall_step % checkpointing_steps == 0:
																														if args.output_dir is not None:
																																			_snake_case							:       int												= os.path.join(args.output_dir	,	snake_case_					)
																														accelerator.save_state(snake_case_					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin					)					)					)
										accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used					)					)
										accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked					)					)
										accelerator.print(
										    "Total Peak Memory consumed during the train (max): {}".format(
										        tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "train_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin					),
															    }	,	step=snake_case_	,	)

										# New Code #
										# context manager to track the peak memory usage during the evaluation
										with TorchTracemalloc() as tracemalloc:
															model.eval()
															for step, batch in enumerate(snake_case_					):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device					)
																				with torch.no_grad():
																									_snake_case							:       Any												= model(**snake_case_					)
																				_snake_case							:       str												= outputs.logits.argmax(dim=-1					)
																				_snake_case							,			_snake_case							:       Any												= accelerator.gather_for_metrics((predictions, batch["labels"])					)
																				metric.add_batch(
																				    predictions=snake_case_	,	references=snake_case_	,	)

															_snake_case							:       Tuple												= metric.compute()
															# Use accelerator.print to print only on the main process.
															accelerator.print(F"""epoch {epoch}:"""	,	snake_case_					)
															if args.with_tracking:
																				accelerator.log(
																				    {
																				        "accuracy": eval_metric["accuracy"],
																				        "f1": eval_metric["f1"],
																				        "train_loss": total_loss.item() / len(snake_case_					),
																				    }	,	step=snake_case_	,	)

															if checkpointing_steps == "epoch":
																				_snake_case							:       Any												= F"""epoch_{epoch}"""
																				if args.output_dir is not None:
																									_snake_case							:       Optional[Any]												= os.path.join(args.output_dir	,	snake_case_					)
																				accelerator.save_state(snake_case_					)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the eval : {}".format(bamb(tracemalloc.begin					)					)					)
										accelerator.print("Memory consumed at the end of the eval (end-begin): {}".format(tracemalloc.used					)					)
										accelerator.print("Peak Memory consumed during the eval (max-begin): {}".format(tracemalloc.peaked					)					)
										accelerator.print(
										    "Total Peak Memory consumed during the eval (max): {}".format(tracemalloc.peaked + bamb(tracemalloc.begin					)					)					)
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "eval_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin					),
															    }	,	step=snake_case_	,	)

					if args.with_tracking:
										accelerator.end_training()







def  A					(				):
					_snake_case							:       Optional[Any]												= argparse.ArgumentParser(description="Simple example of training script."					)
					parser.add_argument(
					    "--mixed_precision"	,	type=snake_case_	,	default=snake_case_	,	choices=["no", "fp16", "bf16", "fp8"]	,	help="Whether to use mixed precision. Choose"
					    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
					    "and an Nvidia Ampere GPU."	,	)
					parser.add_argument("--cpu"	,	action="store_true"	,	help="If passed, will train on the CPU."					)
					parser.add_argument(
					    "--checkpointing_steps"	,	type=snake_case_	,	default=snake_case_	,	help="Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch."	,	)
					parser.add_argument(
					    "--resume_from_checkpoint"	,	type=snake_case_	,	default=snake_case_	,	help="If the training should continue from a checkpoint folder."	,	)
					parser.add_argument(
					    "--with_tracking"	,	action="store_true"	,	help="Whether to load in all available experiment trackers from the environment and use them for logging."	,	)
					parser.add_argument(
					    "--output_dir"	,	type=snake_case_	,	default="."	,	help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."	,	)
					parser.add_argument(
					    "--logging_dir"	,	type=snake_case_	,	default="logs"	,	help="Location on where to store experiment tracking logs`"	,	)
					parser.add_argument(
					    "--model_name_or_path"	,	type=snake_case_	,	help="Path to pretrained model or model identifier from huggingface.co/models."	,	required=snake_case_	,	)
					_snake_case							:       Optional[Any]												= parser.parse_args()
					_snake_case							:       Tuple												= {"lr": 2e-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
					training_function(snake_case_	,	snake_case_					)


if __name__ == "__main__":
							main()





from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


lowercase_      :							List[str]    							=					logging.get_logger(__name__)






class        __lowerCAmelCase					(			nn.Module					):
			snake_case_ :					HybridCLIPConfig
			snake_case_ :					jnp.dtype						=	jnp.floataa







			def UpperCamelCase     (			self       :     Union[str, Any] ):




									"""simple docstring"""


									_UpperCAmelCase						=	self.config.text_config
									_UpperCAmelCase						=	self.config.vision_config

									_UpperCAmelCase						=	self.config.projection_dim
									_UpperCAmelCase						=	text_config.hidden_size
									_UpperCAmelCase						=	vision_config.hidden_size

									_UpperCAmelCase						=	FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
									_UpperCAmelCase						=	FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__       ,       snake_case__ ).module_class

									_UpperCAmelCase						=	text_module(snake_case__       ,       dtype=self.dtype )
									_UpperCAmelCase						=	vision_module(snake_case__       ,       dtype=self.dtype )

									_UpperCAmelCase						=	nn.Dense(
									    self.projection_dim       ,       dtype=self.dtype       ,       kernel_init=jax.nn.initializers.normal(0.02 )       ,       use_bias=snake_case__       ,       )
									_UpperCAmelCase						=	nn.Dense(
									    self.projection_dim       ,       dtype=self.dtype       ,       kernel_init=jax.nn.initializers.normal(0.02 )       ,       use_bias=snake_case__       ,       )
									_UpperCAmelCase						=	self.param("logit_scale"       ,       jax.nn.initializers.ones       ,       [] )







			def __call__(			self       :     List[Any]       ,       snake_case__       :     List[Any]=None       ,       snake_case__       :     List[str]=None       ,       snake_case__       :     Optional[Any]=None       ,       snake_case__       :     Dict=None       ,       snake_case__       :     Tuple=None       ,       snake_case__       :     bool = True       ,       snake_case__       :     Dict=None       ,       snake_case__       :     Dict=None       ,       snake_case__       :     Dict=None       ,       ):




									"""simple docstring"""


									_UpperCAmelCase						=	return_dict if return_dict is not None else self.config.return_dict

									_UpperCAmelCase						=	self.vision_model(
									    pixel_values=snake_case__       ,       deterministic=snake_case__       ,       output_attentions=snake_case__       ,       output_hidden_states=snake_case__       ,       return_dict=snake_case__       ,       )

									_UpperCAmelCase						=	self.text_model(
									    input_ids=snake_case__       ,       attention_mask=snake_case__       ,       token_type_ids=snake_case__       ,       position_ids=snake_case__       ,       deterministic=snake_case__       ,       output_attentions=snake_case__       ,       output_hidden_states=snake_case__       ,       return_dict=snake_case__       ,       )

									_UpperCAmelCase						=	vision_outputs[1]
									_UpperCAmelCase						=	self.visual_projection(snake_case__ )

									_UpperCAmelCase						=	text_outputs[1]
									_UpperCAmelCase						=	self.text_projection(snake_case__ )

									# normalized features
									_UpperCAmelCase						=	image_embeds / jnp.linalg.norm(snake_case__       ,       axis=-1       ,       keepdims=snake_case__ )
									_UpperCAmelCase						=	text_embeds / jnp.linalg.norm(snake_case__       ,       axis=-1       ,       keepdims=snake_case__ )

									# cosine similarity as logits
									_UpperCAmelCase						=	jnp.exp(self.logit_scale )
									_UpperCAmelCase						=	jnp.matmul(snake_case__       ,       image_embeds.T ) * logit_scale
									_UpperCAmelCase						=	logits_per_text.T

									if not return_dict:
															return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

									return FlaxCLIPOutput(
									    logits_per_image=snake_case__       ,       logits_per_text=snake_case__       ,       text_embeds=snake_case__       ,       image_embeds=snake_case__       ,       text_model_output=snake_case__       ,       vision_model_output=snake_case__       ,       )






class        __lowerCAmelCase					(			UpperCAmelCase__					):
			snake_case_ :					Union[str, Any]						=	HybridCLIPConfig
			snake_case_ :					Optional[Any]						=	FlaxHybridCLIPModule







			def __init__(			self       :     List[Any]       ,       snake_case__       :     HybridCLIPConfig       ,       snake_case__       :     Optional[Tuple] = None       ,       snake_case__       :     int = 0       ,       snake_case__       :     jnp.dtype = jnp.floataa       ,       **snake_case__       :     Any       ,       ):




									"""simple docstring"""


									if input_shape is None:
															_UpperCAmelCase						=	((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

									_UpperCAmelCase						=	self.module_class(config=snake_case__       ,       dtype=snake_case__       ,       **snake_case__ )
									super().__init__(snake_case__       ,       snake_case__       ,       input_shape=snake_case__       ,       seed=snake_case__       ,       dtype=snake_case__ )







			def UpperCamelCase     (			self       :     Union[str, Any]       ,       snake_case__       :     jax.random.PRNGKey       ,       snake_case__       :     Tuple       ,       snake_case__       :     FrozenDict = None ):




									"""simple docstring"""


									_UpperCAmelCase						=	jnp.zeros(input_shape[0]       ,       dtype="i4" )
									_UpperCAmelCase						=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case__ ).shape[-1] )       ,       input_shape[0] )
									_UpperCAmelCase						=	jnp.ones_like(snake_case__ )
									_UpperCAmelCase						=	jnp.ones_like(snake_case__ )

									_UpperCAmelCase						=	jax.random.normal(snake_case__       ,       input_shape[1] )

									_UpperCAmelCase				,					_UpperCAmelCase						=	jax.random.split(snake_case__ )
									_UpperCAmelCase						=	{"params": params_rng, "dropout": dropout_rng}

									return self.module.init(snake_case__       ,       snake_case__       ,       snake_case__       ,       snake_case__       ,       snake_case__       ,       snake_case__ )["params"]







			def __call__(			self       :     Union[str, Any]       ,       snake_case__       :     Optional[int]       ,       snake_case__       :     Tuple       ,       snake_case__       :     Optional[int]=None       ,       snake_case__       :     Tuple=None       ,       snake_case__       :     Optional[Any]=None       ,       snake_case__       :     dict = None       ,       snake_case__       :     jax.random.PRNGKey = None       ,       snake_case__       :     bool = False       ,       snake_case__       :     Optional[bool] = None       ,       snake_case__       :     Optional[bool] = None       ,       snake_case__       :     Optional[bool] = None       ,       ):




									"""simple docstring"""


									_UpperCAmelCase						=	output_attentions if output_attentions is not None else self.config.output_attentions
									_UpperCAmelCase						=	(
									    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
									)
									_UpperCAmelCase						=	return_dict if return_dict is not None else self.config.return_dict

									if position_ids is None:
															_UpperCAmelCase						=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case__ ).shape[-1] )       ,       input_ids.shape )

									if token_type_ids is None:
															_UpperCAmelCase						=	jnp.zeros_like(snake_case__ )

									if attention_mask is None:
															_UpperCAmelCase						=	jnp.ones_like(snake_case__ )

									# Handle any PRNG if needed
									_UpperCAmelCase						=	{}
									if dropout_rng is not None:
															_UpperCAmelCase						=	dropout_rng

									return self.module.apply(
									    {"params": params or self.params}       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype=jnp.floataa )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       not train       ,       snake_case__       ,       snake_case__       ,       snake_case__       ,       rngs=snake_case__       ,       )







			def UpperCamelCase     (			self       :     int       ,       snake_case__       :     int       ,       snake_case__       :     List[str]=None       ,       snake_case__       :     Union[str, Any]=None       ,       snake_case__       :     str=None       ,       snake_case__       :     dict = None       ,       snake_case__       :     jax.random.PRNGKey = None       ,       snake_case__       :     int=False       ,       ):




									"""simple docstring"""


									if position_ids is None:
															_UpperCAmelCase						=	jnp.broadcast_to(jnp.arange(jnp.atleast_ad(snake_case__ ).shape[-1] )       ,       input_ids.shape )

									if token_type_ids is None:
															_UpperCAmelCase						=	jnp.zeros_like(snake_case__ )

									if attention_mask is None:
															_UpperCAmelCase						=	jnp.ones_like(snake_case__ )

									# Handle any PRNG if needed
									_UpperCAmelCase						=	{}
									if dropout_rng is not None:
															_UpperCAmelCase						=	dropout_rng

									def _get_features(snake_case__       :     Union[str, Any]       ,       snake_case__       :     Optional[Any]       ,       snake_case__       :     List[Any]       ,       snake_case__       :     int       ,       snake_case__       :     Dict       ,       snake_case__       :     List[str] ):
															_UpperCAmelCase						=	module.text_model(
															    input_ids=snake_case__       ,       attention_mask=snake_case__       ,       position_ids=snake_case__       ,       token_type_ids=snake_case__       ,       deterministic=snake_case__       ,       )
															_UpperCAmelCase						=	text_outputs[1]
															_UpperCAmelCase						=	module.text_projection(snake_case__ )
															return text_features

									return self.module.apply(
									    {"params": params or self.params}       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       jnp.array(snake_case__       ,       dtype="i4" )       ,       not train       ,       method=_get_features       ,       rngs=snake_case__       ,       )







			def UpperCamelCase     (			self       :     List[Any]       ,       snake_case__       :     Optional[Any]       ,       snake_case__       :     dict = None       ,       snake_case__       :     jax.random.PRNGKey = None       ,       snake_case__       :     str=False ):




									"""simple docstring"""


									_UpperCAmelCase						=	{}
									if dropout_rng is not None:
															_UpperCAmelCase						=	dropout_rng

									def _get_features(snake_case__       :     Optional[Any]       ,       snake_case__       :     Tuple       ,       snake_case__       :     int ):
															_UpperCAmelCase						=	module.vision_model(pixel_values=snake_case__       ,       deterministic=snake_case__ )
															_UpperCAmelCase						=	vision_outputs[1]  # pooled_output
															_UpperCAmelCase						=	module.visual_projection(snake_case__ )
															return image_features

									return self.module.apply(
									    {"params": params or self.params}       ,       jnp.array(snake_case__       ,       dtype=jnp.floataa )       ,       not train       ,       method=_get_features       ,       rngs=snake_case__       ,       )







			@classmethod
			def UpperCamelCase     (			cls       :     Tuple       ,       snake_case__       :     str = None       ,       snake_case__       :     str = None       ,       *snake_case__       :     Any       ,       **snake_case__       :     Tuple       ,       ):




									"""simple docstring"""


									_UpperCAmelCase						=	{
									    argument[len("text_" ) :]: value for argument, value in kwargs.items() if argument.startswith("text_" )
									}

									_UpperCAmelCase						=	{
									    argument[len("vision_" ) :]: value for argument, value in kwargs.items() if argument.startswith("vision_" )
									}

									# remove text, vision kwargs from kwargs
									for key in kwargs_text.keys():
															del kwargs["text_" + key]
									for key in kwargs_vision.keys():
															del kwargs["vision_" + key]

									# Load and initialize the text and vision model
									_UpperCAmelCase						=	kwargs_text.pop("model"       ,       snake_case__ )
									if text_model is None:
															assert (
															    text_model_name_or_path is not None
															), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
															from transformers import FlaxAutoModel

															if "config" not in kwargs_text:
																					from transformers import AutoConfig

																					_UpperCAmelCase						=	AutoConfig.from_pretrained(snake_case__ )
																					_UpperCAmelCase						=	text_config

															_UpperCAmelCase						=	FlaxAutoModel.from_pretrained(snake_case__       ,       *snake_case__       ,       **snake_case__ )

									_UpperCAmelCase						=	kwargs_vision.pop("model"       ,       snake_case__ )
									if vision_model is None:
															assert (
															    vision_model_name_or_path is not None
															), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
															from transformers import FlaxAutoModel

															if "config" not in kwargs_vision:
																					from transformers import AutoConfig

																					_UpperCAmelCase						=	AutoConfig.from_pretrained(snake_case__ )
																					_UpperCAmelCase						=	vision_config

															_UpperCAmelCase						=	FlaxAutoModel.from_pretrained(snake_case__       ,       *snake_case__       ,       **snake_case__ )

									# instantiate config with corresponding kwargs
									_UpperCAmelCase						=	kwargs.pop("dtype"       ,       jnp.floataa )
									_UpperCAmelCase						=	HybridCLIPConfig.from_text_vision_configs(text_model.config       ,       vision_model.config       ,       **snake_case__ )

									# init model
									_UpperCAmelCase						=	cls(snake_case__       ,       *snake_case__       ,       dtype=snake_case__       ,       **snake_case__ )

									if vision_config.model_type == "clip":
															_UpperCAmelCase						=	vision_model.params["vision_model"]
															_UpperCAmelCase						=	vision_model.params["visual_projection"]["kernel"]
									else:
															_UpperCAmelCase						=	vision_model.params

									_UpperCAmelCase						=	text_model.params

									return model




