'''simple docstring'''

import argparse
import logging
import os
import sys
import time

import tensorflow as tf
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
from transformers.utils import is_sagemaker_dp_enabled


if os.environ.get('''SDP_ENABLED''') or is_sagemaker_dp_enabled():
							lowercase_  		=			True
							lowercase_  		=			'''p3dn.24xlarge'''
							import smdistributed.dataparallel.tensorflow as sdp
else:
							lowercase_  		=			False



def 				UpperCamelCase__      (      a__      ,   a__      ,   a__      ,   a__      ,   a__      ,   a__      ,   a__=None   ):


						'''simple docstring'''






						_lowerCAmelCase					    =tqdm(a__   )
						for i, batch in enumerate(a__   ):
												with tf.GradientTape() as tape:
																		_lowerCAmelCase    ,					_lowerCAmelCase					    =batch
																		_lowerCAmelCase					    =model(a__   )
																		_lowerCAmelCase					    =loss(a__      ,   outputs.logits   )

												if SDP_ENABLED:
																		_lowerCAmelCase					    =sdp.DistributedGradientTape(a__      ,   sparse_as_dense=a__   )

												_lowerCAmelCase					    =tape.gradient(a__      ,   model.trainable_variables   )
												opt.apply_gradients(zip(a__      ,   model.trainable_variables   )   )

												pbar.set_description(F'''Loss: {loss_value:.4f}'''   )

												if SDP_ENABLED and i == 0:
																		sdp.broadcast_variables(model.variables      ,   root_rank=0   )
																		sdp.broadcast_variables(opt.variables()      ,   root_rank=0   )

												if max_steps and i >= max_steps:
																		break

						_lowerCAmelCase					    ={'loss': loss_value.numpy()}
						return train_results



def 				UpperCamelCase__      (      a__      ,   a__      ,   a__   ):


						'''simple docstring'''






						_lowerCAmelCase    ,					_lowerCAmelCase					    =load_dataset('imdb'      ,   split=['train', 'test']   )

						# Preprocess train dataset
						_lowerCAmelCase					    =train_dataset.map(
						    lambda a__   : tokenizer(e['text']      ,   truncation=a__      ,   padding='max_length'   )      ,   batched=a__   )
						train_dataset.set_format(type='tensorflow'      ,   columns=['input_ids', 'attention_mask', 'label']   )

						_lowerCAmelCase					    ={
						    x: train_dataset[x].to_tensor(default_value=0      ,   shape=[None, tokenizer.model_max_length]   )
						    for x in ['input_ids', 'attention_mask']
						}
						_lowerCAmelCase					    =tf.data.Dataset.from_tensor_slices((train_features, train_dataset['label'])   )

						# Preprocess test dataset
						_lowerCAmelCase					    =test_dataset.map(
						    lambda a__   : tokenizer(e['text']      ,   truncation=a__      ,   padding='max_length'   )      ,   batched=a__   )
						test_dataset.set_format(type='tensorflow'      ,   columns=['input_ids', 'attention_mask', 'label']   )

						_lowerCAmelCase					    ={
						    x: test_dataset[x].to_tensor(default_value=0      ,   shape=[None, tokenizer.model_max_length]   )
						    for x in ['input_ids', 'attention_mask']
						}
						_lowerCAmelCase					    =tf.data.Dataset.from_tensor_slices((test_features, test_dataset['label'])   )

						if SDP_ENABLED:
												_lowerCAmelCase					    =tf_train_dataset.shard(sdp.size()      ,   sdp.rank()   )
												_lowerCAmelCase					    =tf_test_dataset.shard(sdp.size()      ,   sdp.rank()   )
						_lowerCAmelCase					    =tf_train_dataset.batch(a__      ,   drop_remainder=a__   )
						_lowerCAmelCase					    =tf_test_dataset.batch(a__      ,   drop_remainder=a__   )

						return tf_train_dataset, tf_test_dataset


if __name__ == "__main__":
							lowercase_  		=			argparse.ArgumentParser()

							# Hyperparameters sent by the client are passed as command-line arguments to the script.
							parser.add_argument('''--epochs''', type=int, default=3)
							parser.add_argument('''--per_device_train_batch_size''', type=int, default=16)
							parser.add_argument('''--per_device_eval_batch_size''', type=int, default=8)
							parser.add_argument('''--model_name_or_path''', type=str)
							parser.add_argument('''--learning_rate''', type=str, default=5e-5)
							parser.add_argument('''--do_train''', type=bool, default=True)
							parser.add_argument('''--do_eval''', type=bool, default=True)
							parser.add_argument('''--output_dir''', type=str)
							parser.add_argument('''--max_steps''', type=int, default=None)

							# Data, model, and output directories
							parser.add_argument('''--output_data_dir''', type=str, default=os.environ['''SM_OUTPUT_DATA_DIR'''])
							parser.add_argument('''--model_dir''', type=str, default=os.environ['''SM_MODEL_DIR'''])
							parser.add_argument('''--n_gpus''', type=str, default=os.environ['''SM_NUM_GPUS'''])

							lowercase_		,      lowercase_  		=			parser.parse_known_args()

							# Set up logging
							lowercase_  		=			logging.getLogger(__name__)

							logging.basicConfig(
							    level=logging.getLevelName('''INFO'''),
							    handlers=[logging.StreamHandler(sys.stdout)],
							    format='''%(asctime)s - %(name)s - %(levelname)s - %(message)s''',
							)

							if SDP_ENABLED:
														sdp.init()

														lowercase_  		=			tf.config.experimental.list_physical_devices('''GPU''')
														for gpu in gpus:
																					tf.config.experimental.set_memory_growth(gpu, True)
														if gpus:
																					tf.config.experimental.set_visible_devices(gpus[sdp.local_rank()], '''GPU''')

    # Load model and tokenizer
							lowercase_  		=			TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
							lowercase_  		=			AutoTokenizer.from_pretrained(args.model_name_or_path)

							# get datasets
							lowercase_		,      lowercase_  		=			get_datasets(
							    tokenizer=tokenizer,
							    train_batch_size=args.per_device_train_batch_size,
							    eval_batch_size=args.per_device_eval_batch_size,
							)

							# fine optimizer and loss
							lowercase_  		=			tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
							lowercase_  		=			tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
							lowercase_  		=			[tf.keras.metrics.SparseCategoricalAccuracy()]
							model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

							# Training
							if args.do_train:
														# train_results = model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.train_batch_size)
														lowercase_  		=			time.time()
														lowercase_  		=			fit(
														    model,
														    loss,
														    optimizer,
														    tf_train_dataset,
														    args.epochs,
														    args.per_device_train_batch_size,
														    max_steps=args.max_steps,
														)
														lowercase_  		=			time.time() - start_train_time
														logger.info('''*** Train ***''')
														logger.info(F'train_runtime = {end_train_time}')

														lowercase_  		=			os.path.join(args.output_dir, '''train_results.txt''')

														if not SDP_ENABLED or sdp.rank() == 0:
																					with open(output_eval_file, '''w''') as writer:
																												logger.info('''***** Train results *****''')
																												logger.info(train_results)
																												for key, value in train_results.items():
																																			logger.info(F'  {key} = {value}')
																																			writer.write(F'{key} = {value}\n')

    # Evaluation
							if args.do_eval and (not SDP_ENABLED or sdp.rank() == 0):
														lowercase_  		=			model.evaluate(tf_test_dataset, batch_size=args.per_device_eval_batch_size, return_dict=True)
														logger.info('''*** Evaluate ***''')

														lowercase_  		=			os.path.join(args.output_dir, '''eval_results.txt''')

														with open(output_eval_file, '''w''') as writer:
																					logger.info('''***** Eval results *****''')
																					logger.info(result)
																					for key, value in result.items():
																												logger.info(F'  {key} = {value}')
																												writer.write(F'{key} = {value}\n')

    # Save result
							if SDP_ENABLED:
														if sdp.rank() == 0:
																					model.save_pretrained(args.output_dir)
																					tokenizer.save_pretrained(args.output_dir)
							else:
														model.save_pretrained(args.output_dir)
														tokenizer.save_pretrained(args.output_dir)






'''simple docstring'''

import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_url
from PIL import Image

from transformers import DeformableDetrConfig, DeformableDetrForObjectDetection, DeformableDetrImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
lowercase_  		=			logging.get_logger(__name__)



def 				UpperCamelCase__      (      a__   ):


						'''simple docstring'''






						if "backbone.0.body" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('backbone.0.body'      ,   'backbone.conv_encoder.model'   )
						if "transformer" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('transformer.'      ,   ''   )
						if "norm1" in orig_key:
												if "encoder" in orig_key:
																		_lowerCAmelCase					    =orig_key.replace('norm1'      ,   'self_attn_layer_norm'   )
												else:
																		_lowerCAmelCase					    =orig_key.replace('norm1'      ,   'encoder_attn_layer_norm'   )
						if "norm2" in orig_key:
												if "encoder" in orig_key:
																		_lowerCAmelCase					    =orig_key.replace('norm2'      ,   'final_layer_norm'   )
												else:
																		_lowerCAmelCase					    =orig_key.replace('norm2'      ,   'self_attn_layer_norm'   )
						if "norm3" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('norm3'      ,   'final_layer_norm'   )
						if "linear1" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('linear1'      ,   'fc1'   )
						if "linear2" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('linear2'      ,   'fc2'   )
						if "query_embed" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('query_embed'      ,   'query_position_embeddings'   )
						if "cross_attn" in orig_key:
												_lowerCAmelCase					    =orig_key.replace('cross_attn'      ,   'encoder_attn'   )

						return orig_key



def 				UpperCamelCase__      (      a__   ):


						'''simple docstring'''






						for i in range(6   ):
												# read in weights + bias of input projection layer of self-attention
												_lowerCAmelCase					    =state_dict.pop(F'''decoder.layers.{i}.self_attn.in_proj_weight'''   )
												_lowerCAmelCase					    =state_dict.pop(F'''decoder.layers.{i}.self_attn.in_proj_bias'''   )
												# next, add query, keys and values (in that order) to the state dict
												_lowerCAmelCase					    =in_proj_weight[:2_5_6, :]
												_lowerCAmelCase					    =in_proj_bias[:2_5_6]
												_lowerCAmelCase					    =in_proj_weight[2_5_6:5_1_2, :]
												_lowerCAmelCase					    =in_proj_bias[2_5_6:5_1_2]
												_lowerCAmelCase					    =in_proj_weight[-2_5_6:, :]
												_lowerCAmelCase					    =in_proj_bias[-2_5_6:]



def 				UpperCamelCase__      (      ):


						'''simple docstring'''






						_lowerCAmelCase					    ='http://images.cocodataset.org/val2017/000000039769.jpg'
						_lowerCAmelCase					    =Image.open(requests.get(a__      ,   stream=a__   ).raw   )

						return im



@torch.no_grad()
def 				UpperCamelCase__      (      a__      ,   a__      ,   a__      ,   a__      ,   a__      ,   a__      ,   a__      ,   ):


						'''simple docstring'''






						_lowerCAmelCase					    =DeformableDetrConfig()
						# set config attributes
						if single_scale:
												_lowerCAmelCase					    =1
						_lowerCAmelCase					    =dilation
						_lowerCAmelCase					    =with_box_refine
						_lowerCAmelCase					    =two_stage
						# set labels
						_lowerCAmelCase					    =9_1
						_lowerCAmelCase					    ='huggingface/label-files'
						_lowerCAmelCase					    ='coco-detection-id2label.json'
						_lowerCAmelCase					    =json.load(open(cached_download(hf_hub_url(a__      ,   a__      ,   repo_type='dataset'   )   )      ,   'r'   )   )
						_lowerCAmelCase					    ={int(a__   ): v for k, v in idalabel.items()}
						_lowerCAmelCase					    =idalabel
						_lowerCAmelCase					    ={v: k for k, v in idalabel.items()}

						# load image processor
						_lowerCAmelCase					    =DeformableDetrImageProcessor(format='coco_detection'   )

						# prepare image
						_lowerCAmelCase					    =prepare_img()
						_lowerCAmelCase					    =image_processor(images=a__      ,   return_tensors='pt'   )
						_lowerCAmelCase					    =encoding['pixel_values']

						logger.info('Converting model...'   )

						# load original state dict
						_lowerCAmelCase					    =torch.load(a__      ,   map_location='cpu'   )['model']
						# rename keys
						for key in state_dict.copy().keys():
												_lowerCAmelCase					    =state_dict.pop(a__   )
												_lowerCAmelCase					    =val
						# query, key and value matrices need special treatment
						read_in_q_k_v(a__   )
						# important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
						_lowerCAmelCase					    ='model.'
						for key in state_dict.copy().keys():
												if not key.startswith('class_embed'   ) and not key.startswith('bbox_embed'   ):
																		_lowerCAmelCase					    =state_dict.pop(a__   )
																		_lowerCAmelCase					    =val
    # finally, create HuggingFace model and load state dict
						_lowerCAmelCase					    =DeformableDetrForObjectDetection(a__   )
						model.load_state_dict(a__   )
						model.eval()

						_lowerCAmelCase					    ='cuda' if torch.cuda.is_available() else 'cpu'
						model.to(a__   )
						# verify our conversion
						_lowerCAmelCase					    =model(pixel_values.to(a__   )   )

						_lowerCAmelCase					    =torch.tensor(
						    [[-9.6_645, -4.3_449, -5.8_705], [-9.7_035, -3.8_504, -5.0_724], [-10.5_634, -5.3_379, -7.5_116]]   )
						_lowerCAmelCase					    =torch.tensor([[0.8_693, 0.2_289, 0.2_492], [0.3_150, 0.5_489, 0.5_845], [0.5_563, 0.7_580, 0.8_518]]   )

						if single_scale:
												_lowerCAmelCase					    =torch.tensor(
												    [[-9.9_051, -4.2_541, -6.4_852], [-9.6_947, -4.0_854, -6.8_033], [-10.0_665, -5.8_470, -7.7_003]]   )
												_lowerCAmelCase					    =torch.tensor([[0.7_292, 0.4_991, 0.5_532], [0.7_959, 0.2_426, 0.4_236], [0.7_582, 0.3_518, 0.4_451]]   )

						if single_scale and dilation:
												_lowerCAmelCase					    =torch.tensor(
												    [[-8.9_652, -4.1_074, -5.6_635], [-9.0_596, -4.9_447, -6.6_075], [-10.1_178, -4.5_275, -6.2_671]]   )
												_lowerCAmelCase					    =torch.tensor([[0.7_665, 0.4_130, 0.4_769], [0.8_364, 0.1_841, 0.3_391], [0.6_261, 0.3_895, 0.7_978]]   )

						if with_box_refine:
												_lowerCAmelCase					    =torch.tensor(
												    [[-8.8_895, -5.4_187, -6.8_153], [-8.4_706, -6.1_668, -7.6_184], [-9.0_042, -5.5_359, -6.9_141]]   )
												_lowerCAmelCase					    =torch.tensor([[0.7_828, 0.2_208, 0.4_323], [0.0_892, 0.5_996, 0.1_319], [0.5_524, 0.6_389, 0.8_914]]   )

						if with_box_refine and two_stage:
												_lowerCAmelCase					    =torch.tensor(
												    [[-6.7_108, -4.3_213, -6.3_777], [-8.9_014, -6.1_799, -6.7_240], [-6.9_315, -4.4_735, -6.2_298]]   )
												_lowerCAmelCase					    =torch.tensor([[0.2_583, 0.5_499, 0.4_683], [0.7_652, 0.9_068, 0.4_882], [0.5_490, 0.2_763, 0.0_564]]   )

						print('Logits:'      ,   outputs.logits[0, :3, :3]   )

						assert torch.allclose(outputs.logits[0, :3, :3]      ,   expected_logits.to(a__   )      ,   atol=1E-4   )
						assert torch.allclose(outputs.pred_boxes[0, :3, :3]      ,   expected_boxes.to(a__   )      ,   atol=1E-4   )

						print('Everything ok!'   )

						# Save model and image processor
						logger.info(F'''Saving PyTorch model and image processor to {pytorch_dump_folder_path}...'''   )
						Path(a__   ).mkdir(exist_ok=a__   )
						model.save_pretrained(a__   )
						image_processor.save_pretrained(a__   )

						# Push to hub
						if push_to_hub:
												_lowerCAmelCase					    ='deformable-detr'
												model_name += "-single-scale" if single_scale else ""
												model_name += "-dc5" if dilation else ""
												model_name += "-with-box-refine" if with_box_refine else ""
												model_name += "-two-stage" if two_stage else ""
												print('Pushing model to hub...'   )
												model.push_to_hub(repo_path_or_name=a__      ,   organization='nielsr'      ,   commit_message='Add model'   )


if __name__ == "__main__":
							lowercase_  		=			argparse.ArgumentParser()

							parser.add_argument(
							    '''--checkpoint_path''',
							    type=str,
							    default='''/home/niels/checkpoints/deformable_detr/r50_deformable_detr-checkpoint.pth''',
							    help='''Path to Pytorch checkpoint (.pth file) you\'d like to convert.''',
							)
							parser.add_argument('''--single_scale''', action='''store_true''', help='''Whether to set config.num_features_levels = 1.''')
							parser.add_argument('''--dilation''', action='''store_true''', help='''Whether to set config.dilation=True.''')
							parser.add_argument('''--with_box_refine''', action='''store_true''', help='''Whether to set config.with_box_refine=True.''')
							parser.add_argument('''--two_stage''', action='''store_true''', help='''Whether to set config.two_stage=True.''')
							parser.add_argument(
							    '''--pytorch_dump_folder_path''',
							    default=None,
							    type=str,
							    required=True,
							    help='''Path to the folder to output PyTorch model.''',
							)
							parser.add_argument(
							    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
							)
							lowercase_  		=			parser.parse_args()
							convert_deformable_detr_checkpoint(
							    args.checkpoint_path,
							    args.single_scale,
							    args.dilation,
							    args.with_box_refine,
							    args.two_stage,
							    args.pytorch_dump_folder_path,
							    args.push_to_hub,
							)





