




"""simple docstring"""

import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
		import torch

		from ..models.auto.modeling_auto import (
		    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
		    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
		)

if is_tf_available() and is_tensorflow_probability_available():
		import tensorflow as tf
		import tensorflow_probability as tfp

		from ..models.auto.modeling_tf_auto import (
		    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
		)
class      A__   (   lowercase_  ):




	def __call__( self	:						Dict   ,     lowercase	:						List[Any]=None   ,     lowercase	:						Dict=None   ,     **lowercase	:						Union[str, Any]						)       ->			Tuple:
						# Returns tqa_pipeline_inputs of shape:
						# [
						#   {"table": pd.DataFrame, "query": List[str]},
						#   ...,
						#   {"table": pd.DataFrame, "query" : List[str]}
						# ]
						requires_backends(self   ,     """pandas"""						)
						import pandas as pd

						if table is None:
											raise ValueError("""Keyword argument `table` cannot be None."""						)
						elif query is None:
											if isinstance(lowercase   ,     lowercase						) and table.get("""query"""						) is not None and table.get("""table"""						) is not None:
																__snake_case        =						[table]
											elif isinstance(lowercase   ,     lowercase						) and len(lowercase						) > 0:
																if not all(isinstance(lowercase   ,     lowercase						) for d in table						):
																					raise ValueError(
																					    f'''Keyword argument `table` should be a list of dict, but is {(type(lowercase						) for d in table)}'''						)

																if table[0].get("""query"""						) is not None and table[0].get("""table"""						) is not None:
																					__snake_case        =						table
																else:
																					raise ValueError(
																					    """If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"""
																					    f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''						)
											elif Dataset is not None and isinstance(lowercase   ,     lowercase						) or isinstance(lowercase   ,     types.GeneratorType						):
																return table
											else:
																raise ValueError(
																    """Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but """
																    f'''is {type(lowercase						)})'''						)
						else:
											__snake_case        =						[{"""table""": table, """query""": query}]

						for tqa_pipeline_input in tqa_pipeline_inputs:
											if not isinstance(tqa_pipeline_input["""table"""]   ,     pd.DataFrame						):
																if tqa_pipeline_input["table"] is None:
																					raise ValueError("""Table cannot be None."""						)

																__snake_case        =						pd.DataFrame(tqa_pipeline_input["""table"""]						)

						return tqa_pipeline_inputs

@add_end_docstrings(lowercase_  )
class      A__   (   lowercase_  ):
	A_				:					Optional[Any]      =   """table,query"""




	def __init__( self	:						int   ,     lowercase	:						Optional[Any]=TableQuestionAnsweringArgumentHandler()   ,     *lowercase	:						Dict   ,     **lowercase	:						Optional[Any]						)       ->			str:
						super().__init__(*lowercase   ,     **lowercase						)
						__snake_case        =						args_parser

						self.check_model_type(
						    dict(
						        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
						        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()						)
						    if self.framework == """tf"""
						    else dict(
						        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()						)						)

						__snake_case        =						bool(getattr(self.model.config   ,     """aggregation_labels"""   ,     lowercase						)						) and bool(
						    getattr(self.model.config   ,     """num_aggregation_labels"""   ,     lowercase						)						)
						__snake_case        =						"""tapas""" if hasattr(self.model.config   ,     """aggregation_labels"""						) else None




	def 				a_		( self	:						List[str]   ,     **lowercase	:						int						)       ->			Optional[Any]:
						return self.model(**lowercase						)




	def 				a_		( self	:						Dict   ,     **lowercase	:						Optional[int]						)       ->			List[str]:
						if self.framework == "pt":
											__snake_case        =						[]
											__snake_case        =						[]
											__snake_case        =						None
											__snake_case        =						inputs["""input_ids"""].shape[0]

											__snake_case        =						inputs["""input_ids"""].to(self.device						)
											__snake_case        =						inputs["""attention_mask"""].to(self.device						)
											__snake_case        =						inputs["""token_type_ids"""].to(self.device						)
											__snake_case        =						None

											for index in range(lowercase						):
																# If sequences have already been processed, the token type IDs will be created according to the previous
																# answer.
																if prev_answers is not None:
																					__snake_case        =						token_type_ids_example[:, 3]  # shape (seq_len,)
																					__snake_case        =						np.zeros_like(prev_labels_example.cpu().numpy()						)  # shape (seq_len,)

																					__snake_case        =						token_type_ids[index]  # shape (seq_len, 7)
																					for i in range(model_labels.shape[0]						):
																										__snake_case        =						token_type_ids_example[:, 0].tolist()[i]
																										__snake_case        =						token_type_ids_example[:, 1].tolist()[i] - 1
																										__snake_case        =						token_type_ids_example[:, 2].tolist()[i] - 1

																										if row_id >= 0 and col_id >= 0 and segment_id == 1:
																															__snake_case        =						int(prev_answers[(col_id, row_id)]						)

																					__snake_case        =						torch.from_numpy(lowercase						).type(torch.long						).to(self.device						)

																__snake_case        =						input_ids[index]
																__snake_case        =						attention_mask[index]  # shape (seq_len,)
																__snake_case        =						token_type_ids[index]  # shape (seq_len, 7)
																__snake_case        =						self.model(
																    input_ids=input_ids_example.unsqueeze(0						)   ,     attention_mask=attention_mask_example.unsqueeze(0						)   ,     token_type_ids=token_type_ids_example.unsqueeze(0						)   ,     )
																__snake_case        =						outputs.logits

																if self.aggregate:
																					all_aggregations.append(outputs.logits_aggregation						)

																all_logits.append(lowercase						)

																__snake_case        =						torch.distributions.Bernoulli(logits=lowercase						)
																__snake_case        =						dist_per_token.probs * attention_mask_example.type(torch.floataa						).to(
																    dist_per_token.probs.device						)

																__snake_case        =						collections.defaultdict(lowercase						)
																for i, p in enumerate(probabilities.squeeze().tolist()						):
																					__snake_case        =						token_type_ids_example[:, 0].tolist()[i]
																					__snake_case        =						token_type_ids_example[:, 1].tolist()[i] - 1
																					__snake_case        =						token_type_ids_example[:, 2].tolist()[i] - 1
																					if col >= 0 and row >= 0 and segment_id == 1:
																										coords_to_probs[(col, row)].append(lowercase						)

																__snake_case        =						{key: np.array(coords_to_probs[key]						).mean() > 0.5 for key in coords_to_probs}

											__snake_case        =						torch.cat(tuple(lowercase						)   ,     0						)

											return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(lowercase						)   ,     0						))
						else:
											__snake_case        =						[]
											__snake_case        =						[]
											__snake_case        =						None
											__snake_case        =						inputs["""input_ids"""].shape[0]

											__snake_case        =						inputs["""input_ids"""]
											__snake_case        =						inputs["""attention_mask"""]
											__snake_case        =						inputs["""token_type_ids"""].numpy()
											__snake_case        =						None

											for index in range(lowercase						):
																# If sequences have already been processed, the token type IDs will be created according to the previous
																# answer.
																if prev_answers is not None:
																					__snake_case        =						token_type_ids_example[:, 3]  # shape (seq_len,)
																					__snake_case        =						np.zeros_like(lowercase   ,     dtype=np.intaa						)  # shape (seq_len,)

																					__snake_case        =						token_type_ids[index]  # shape (seq_len, 7)
																					for i in range(model_labels.shape[0]						):
																										__snake_case        =						token_type_ids_example[:, 0].tolist()[i]
																										__snake_case        =						token_type_ids_example[:, 1].tolist()[i] - 1
																										__snake_case        =						token_type_ids_example[:, 2].tolist()[i] - 1

																										if row_id >= 0 and col_id >= 0 and segment_id == 1:
																															__snake_case        =						int(prev_answers[(col_id, row_id)]						)

																					__snake_case        =						model_labels

																__snake_case        =						input_ids[index]
																__snake_case        =						attention_mask[index]  # shape (seq_len,)
																__snake_case        =						token_type_ids[index]  # shape (seq_len, 7)
																__snake_case        =						self.model(
																    input_ids=np.expand_dims(lowercase   ,     axis=0						)   ,     attention_mask=np.expand_dims(lowercase   ,     axis=0						)   ,     token_type_ids=np.expand_dims(lowercase   ,     axis=0						)   ,     )
																__snake_case        =						outputs.logits

																if self.aggregate:
																					all_aggregations.append(outputs.logits_aggregation						)

																all_logits.append(lowercase						)

																__snake_case        =						tfp.distributions.Bernoulli(logits=lowercase						)
																__snake_case        =						dist_per_token.probs_parameter() * tf.cast(lowercase   ,     tf.floataa						)

																__snake_case        =						collections.defaultdict(lowercase						)
																__snake_case        =						token_type_ids_example
																for i, p in enumerate(tf.squeeze(lowercase						).numpy().tolist()						):
																					__snake_case        =						token_type_ids_example[:, 0].tolist()[i]
																					__snake_case        =						token_type_ids_example[:, 1].tolist()[i] - 1
																					__snake_case        =						token_type_ids_example[:, 2].tolist()[i] - 1
																					if col >= 0 and row >= 0 and segment_id == 1:
																										coords_to_probs[(col, row)].append(lowercase						)

																__snake_case        =						{key: np.array(coords_to_probs[key]						).mean() > 0.5 for key in coords_to_probs}

											__snake_case        =						tf.concat(tuple(lowercase						)   ,     0						)

											return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(lowercase						)   ,     0						))




	def __call__( self	:						List[Any]   ,     *lowercase	:						Union[str, Any]   ,     **lowercase	:						Tuple						)       ->			Dict:
						__snake_case        =						self._args_parser(*lowercase   ,     **lowercase						)

						__snake_case        =						super().__call__(lowercase   ,     **lowercase						)
						if len(lowercase						) == 1:
											return results[0]
						return results




	def 				a_		( self	:						Union[str, Any]   ,     lowercase	:						str=None   ,     lowercase	:						Union[str, Any]=None   ,     lowercase	:						Dict=None   ,     **lowercase	:						Dict						)       ->			Optional[Any]:
						__snake_case        =						{}
						if padding is not None:
											__snake_case        =						padding
						if truncation is not None:
											__snake_case        =						truncation

						__snake_case        =						{}
						if sequential is not None:
											__snake_case        =						sequential
						return preprocess_params, forward_params, {}




	def 				a_		( self	:						Any   ,     lowercase	:						List[Any]   ,     lowercase	:						List[Any]=None   ,     lowercase	:						Optional[int]=True   ,     lowercase	:						List[Any]=None						)       ->			Dict:
						if truncation is None:
											if self.type == "tapas":
																__snake_case        =						"""drop_rows_to_fit"""
											else:
																__snake_case        =						"""do_not_truncate"""

						__snake_case   ,   __snake_case        =						pipeline_input["""table"""], pipeline_input["""query"""]
						if table.empty:
											raise ValueError("""table is empty"""						)
						if query is None or query == "":
											raise ValueError("""query is empty"""						)
						__snake_case        =						self.tokenizer(lowercase   ,     lowercase   ,     return_tensors=self.framework   ,     truncation=lowercase   ,     padding=lowercase						)
						__snake_case        =						table
						return inputs




	def 				a_		( self	:						Optional[Any]   ,     lowercase	:						Any   ,     lowercase	:						Optional[int]=False						)       ->			int:
						__snake_case        =						model_inputs.pop("""table"""						)

						if self.type == "tapas":
											if sequential:
																__snake_case        =						self.sequential_inference(**lowercase						)
											else:
																__snake_case        =						self.batch_inference(**lowercase						)
						else:
											__snake_case        =						self.model.generate(**lowercase						)
						__snake_case        =						{"""model_inputs""": model_inputs, """table""": table, """outputs""": outputs}
						return model_outputs




	def 				a_		( self	:						Dict   ,     lowercase	:						int						)       ->			Optional[int]:
						__snake_case        =						model_outputs["""model_inputs"""]
						__snake_case        =						model_outputs["""table"""]
						__snake_case        =						model_outputs["""outputs"""]
						if self.type == "tapas":
											if self.aggregate:
																__snake_case   ,   __snake_case        =						outputs[:2]
																__snake_case        =						self.tokenizer.convert_logits_to_predictions(lowercase   ,     lowercase   ,     lowercase						)
																__snake_case   ,   __snake_case        =						predictions
																__snake_case        =						{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(lowercase						)}

																__snake_case        =						self.model.config.no_aggregation_label_index
																__snake_case        =						{
																    i: aggregators[i] + """ > """ for i, pred in enumerate(lowercase						) if pred != no_agg_label_index
																}
											else:
																__snake_case        =						outputs[0]
																__snake_case        =						self.tokenizer.convert_logits_to_predictions(lowercase   ,     lowercase						)
																__snake_case        =						predictions[0]
																__snake_case        =						{}
																__snake_case        =						{}
											__snake_case        =						[]
											for index, coordinates in enumerate(lowercase						):
																__snake_case        =						[table.iat[coordinate] for coordinate in coordinates]
																__snake_case        =						aggregators.get(lowercase   ,     """"""						)
																__snake_case        =						aggregators_prefix.get(lowercase   ,     """"""						)
																__snake_case        =						{
																    """answer""": aggregator_prefix + """, """.join(lowercase						),
																    """coordinates""": coordinates,
																    """cells""": [table.iat[coordinate] for coordinate in coordinates],
																}
																if aggregator:
																					__snake_case        =						aggregator

																answers.append(lowercase						)
											if len(lowercase						) == 0:
																raise PipelineException("""Empty answer"""						)
						else:
											__snake_case        =						[{"""answer""": answer} for answer in self.tokenizer.batch_decode(lowercase   ,     skip_special_tokens=lowercase						)]

						return answers if len(lowercase						) > 1 else answers[0]








"""simple docstring"""

import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
		import tensorflow as tf

		tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
		import torch

if is_datasets_available():
		from datasets import load_dataset


UpperCamelCase__			:     str			       =				5e-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors




def    snake_case__						(							lowerCAmelCase :		Namespace		):
					return PTtoTFCommand(
					    args.model_name					,       args.local_dir					,       args.max_error					,       args.new_weights					,       args.no_pr					,       args.push					,       args.extra_commit_description					,       args.override_model_class					,       )

class      A__   (   lowercase_  ):




	@staticmethod
	def 				a_		( lowercase	:						ArgumentParser						)       ->			List[Any]:
						__snake_case        =						parser.add_parser(
						    """pt-to-tf"""   ,     help=(
						        """CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."""
						        """ Can also be used to validate existing weights without opening PRs, with --no-pr."""
						    )   ,     )
						train_parser.add_argument(
						    """--model-name"""   ,     type=lowercase   ,     required=lowercase   ,     help="""The model name, including owner/organization, as seen on the hub."""   ,     )
						train_parser.add_argument(
						    """--local-dir"""   ,     type=lowercase   ,     default=""""""   ,     help="""Optional local directory of the model repository. Defaults to /tmp/{model_name}"""   ,     )
						train_parser.add_argument(
						    """--max-error"""   ,     type=lowercase   ,     default=lowercase   ,     help=(
						        f'''Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk.'''
						    )   ,     )
						train_parser.add_argument(
						    """--new-weights"""   ,     action="""store_true"""   ,     help="""Optional flag to create new TensorFlow weights, even if they already exist."""   ,     )
						train_parser.add_argument(
						    """--no-pr"""   ,     action="""store_true"""   ,     help="""Optional flag to NOT open a PR with converted weights."""						)
						train_parser.add_argument(
						    """--push"""   ,     action="""store_true"""   ,     help="""Optional flag to push the weights directly to `main` (requires permissions)"""   ,     )
						train_parser.add_argument(
						    """--extra-commit-description"""   ,     type=lowercase   ,     default=""""""   ,     help="""Optional additional commit description to use when opening a PR (e.g. to tag the owner)."""   ,     )
						train_parser.add_argument(
						    """--override-model-class"""   ,     type=lowercase   ,     default=lowercase   ,     help="""If you think you know better than the auto-detector, you can specify the model class here. """
						    """Can be either an AutoModel class or a specific model class like BertForSequenceClassification."""   ,     )
						train_parser.set_defaults(func=lowercase						)




	@staticmethod
	def 				a_		( lowercase	:						Optional[Any]   ,     lowercase	:						Any						)       ->			Optional[Any]:
						__snake_case        =						set(pt_outputs.keys()						)
						__snake_case        =						set(tf_outputs.keys()						)
						if pt_out_attrs != tf_out_attrs:
											raise ValueError(
											    f'''The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:'''
											    f''' {tf_out_attrs})'''						)

						# 2. For each output attribute, computes the difference
						def _find_pt_tf_differences(lowercase	:						Optional[int]   ,     lowercase	:						Any   ,     lowercase	:						List[str]   ,     lowercase	:						Tuple=""						):
											# If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
											# recursivelly, keeping the name of the attribute.
											if isinstance(lowercase   ,     torch.Tensor						):
																__snake_case        =						np.max(np.abs(pt_out.numpy() - tf_out.numpy()						)						)
																__snake_case        =						tensor_difference
											else:
																__snake_case        =						attr_name
																for i, pt_item in enumerate(lowercase						):
																					# If it is a named attribute, we keep the name. Otherwise, just its index.
																					if isinstance(lowercase   ,     lowercase						):
																										__snake_case        =						root_name + pt_item
																										__snake_case        =						tf_out[pt_item]
																										__snake_case        =						pt_out[pt_item]
																					else:
																										__snake_case        =						root_name + f'''[{i}]'''
																										__snake_case        =						tf_out[i]
																					__snake_case        =						_find_pt_tf_differences(lowercase   ,     lowercase   ,     lowercase   ,     lowercase						)

											return differences

						return _find_pt_tf_differences(lowercase   ,     lowercase   ,     {}						)




	def __init__( self	:						Tuple   ,     lowercase	:						str   ,     lowercase	:						str   ,     lowercase	:						float   ,     lowercase	:						bool   ,     lowercase	:						bool   ,     lowercase	:						bool   ,     lowercase	:						str   ,     lowercase	:						str   ,     *lowercase	:						Dict   ,     )       ->			Dict:
						__snake_case        =						logging.get_logger("""transformers-cli/pt_to_tf"""						)
						__snake_case        =						model_name
						__snake_case        =						local_dir if local_dir else os.path.join("""/tmp"""   ,     lowercase						)
						__snake_case        =						max_error
						__snake_case        =						new_weights
						__snake_case        =						no_pr
						__snake_case        =						push
						__snake_case        =						extra_commit_description
						__snake_case        =						override_model_class




	def 				a_		( self	:						List[str]   ,     lowercase	:						str   ,     lowercase	:						Dict   ,     lowercase	:						List[str]						)       ->			Optional[Any]:

						def _get_audio_input():
											__snake_case        =						load_dataset("""hf-internal-testing/librispeech_asr_dummy"""   ,     """clean"""   ,     split="""validation"""						)
											__snake_case        =						ds.sort("""id"""						).select(range(2						)						)[:2]["""audio"""]
											__snake_case        =						[x["""array"""] for x in speech_samples]
											return raw_samples

						__snake_case        =						type(pt_model.config						)
						if model_config_class in PROCESSOR_MAPPING:
											__snake_case        =						AutoProcessor.from_pretrained(self._local_dir						)
											if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
																__snake_case        =						processor.tokenizer.eos_token
						elif model_config_class in IMAGE_PROCESSOR_MAPPING:
											__snake_case        =						AutoImageProcessor.from_pretrained(self._local_dir						)
						elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
											__snake_case        =						AutoFeatureExtractor.from_pretrained(self._local_dir						)
						elif model_config_class in TOKENIZER_MAPPING:
											__snake_case        =						AutoTokenizer.from_pretrained(self._local_dir						)
											if processor.pad_token is None:
																__snake_case        =						processor.eos_token
						else:
											raise ValueError(f'''Unknown data processing type (model config type: {model_config_class})'''						)

						__snake_case        =						set(inspect.signature(pt_model.forward						).parameters.keys()						)
						__snake_case        =						{}
						if "input_ids" in model_forward_signature:
											processor_inputs.update(
											    {
											        """text""": ["""Hi there!""", """I am a batch with more than one row and different input lengths."""],
											        """padding""": True,
											        """truncation""": True,
											    }						)
						if "pixel_values" in model_forward_signature:
											__snake_case        =						load_dataset("""cifar10"""   ,     """plain_text"""   ,     split="""test"""						)[:2]["""img"""]
											processor_inputs.update({"""images""": sample_images}						)
						if "input_features" in model_forward_signature:
											__snake_case        =						inspect.signature(processor.feature_extractor						).parameters
											# Pad to the largest input length by default but take feature extractor default
											# padding value if it exists e.g. "max_length" and is not False or None
											if "padding" in feature_extractor_signature:
																__snake_case        =						feature_extractor_signature["""padding"""].default
																if default_strategy is not False and default_strategy is not None:
																					__snake_case        =						default_strategy
																else:
																					__snake_case        =						True
											else:
																__snake_case        =						True
											processor_inputs.update({"""audio""": _get_audio_input(), """padding""": padding_strategy}						)
						if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
											processor_inputs.update({"""audio""": _get_audio_input(), """padding""": True}						)
						__snake_case        =						processor(**lowercase   ,     return_tensors="""pt"""						)
						__snake_case        =						processor(**lowercase   ,     return_tensors="""tf"""						)

						# Extra input requirements, in addition to the input modality
						if (
						    config.is_encoder_decoder
						    or (hasattr(lowercase   ,     """encoder"""						) and hasattr(lowercase   ,     """decoder"""						))
						    or "decoder_input_ids" in tf_dummy_inputs
						):
											__snake_case        =						np.asarray([[1], [1]]   ,     dtype=lowercase						) * (pt_model.config.decoder_start_token_id or 0)
											pt_input.update({"""decoder_input_ids""": torch.tensor(lowercase						)}						)
											tf_input.update({"""decoder_input_ids""": tf.convert_to_tensor(lowercase						)}						)

						return pt_input, tf_input




	def 				a_		( self	:						Optional[Any]						)       ->			Dict:
						# hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
						if version.parse(huggingface_hub.__version__						) < version.parse("""0.9.0"""						):
											raise ImportError(
											    """The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"""
											    """ installation."""						)
						else:
											from huggingface_hub import Repository, create_commit
											from huggingface_hub._commit_api import CommitOperationAdd

						# Fetch remote data
						__snake_case        =						Repository(local_dir=self._local_dir   ,     clone_from=self._model_name						)

						# Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
						__snake_case        =						AutoConfig.from_pretrained(self._local_dir						)
						__snake_case        =						config.architectures
						if self._override_model_class is not None:
											if self._override_model_class.startswith("""TF"""						):
																__snake_case        =						[self._override_model_class[2:]]
											else:
																__snake_case        =						[self._override_model_class]
											try:
																__snake_case        =						getattr(import_module("""transformers"""						)   ,     architectures[0]						)
											except AttributeError:
																raise ValueError(f'''Model class {self._override_model_class} not found in transformers.'''						)
											try:
																__snake_case        =						getattr(import_module("""transformers"""						)   ,     """TF""" + architectures[0]						)
											except AttributeError:
																raise ValueError(f'''TF model class TF{self._override_model_class} not found in transformers.'''						)
						elif architectures is None:  # No architecture defined -- use auto classes
											__snake_case        =						getattr(import_module("""transformers"""						)   ,     """AutoModel"""						)
											__snake_case        =						getattr(import_module("""transformers"""						)   ,     """TFAutoModel"""						)
											self._logger.warning("""No detected architecture, using AutoModel/TFAutoModel"""						)
						else:  # Architecture defined -- use it
											if len(lowercase						) > 1:
																raise ValueError(f'''More than one architecture was found, aborting. (architectures = {architectures})'''						)
											self._logger.warning(f'''Detected architecture: {architectures[0]}'''						)
											__snake_case        =						getattr(import_module("""transformers"""						)   ,     architectures[0]						)
											try:
																__snake_case        =						getattr(import_module("""transformers"""						)   ,     """TF""" + architectures[0]						)
											except AttributeError:
																raise AttributeError(f'''The TensorFlow equivalent of {architectures[0]} doesn\'t exist in transformers.'''						)

        # Check the TF dummy inputs to see what keys we need in the forward pass
						__snake_case        =						tf_class.from_config(lowercase						)
						__snake_case        =						tf_from_pt_model.dummy_inputs

						del tf_from_pt_model  # Try to keep only one model in memory at a time

						# Load the model and get some basic inputs
						__snake_case        =						pt_class.from_pretrained(self._local_dir						)
						pt_model.eval()

						__snake_case   ,   __snake_case        =						self.get_inputs(lowercase   ,     lowercase   ,     lowercase						)

						with torch.no_grad():
											__snake_case        =						pt_model(**lowercase   ,     output_hidden_states=lowercase						)
						del pt_model  # will no longer be used, and may have a large memory footprint

						__snake_case        =						tf_class.from_pretrained(self._local_dir   ,     from_pt=lowercase						)
						__snake_case        =						tf_from_pt_model(**lowercase   ,     output_hidden_states=lowercase   ,     training=lowercase						)

						# Confirms that cross loading PT weights into TF worked.
						__snake_case        =						self.find_pt_tf_differences(lowercase   ,     lowercase						)
						__snake_case        =						{k: v for k, v in crossload_differences.items() if """hidden""" not in k}
						__snake_case        =						{k: v for k, v in crossload_differences.items() if """hidden""" in k}
						if len(lowercase						) == 0 and architectures is not None:
											raise ValueError(
											    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
											    """ output was found. All outputs start with 'hidden'"""						)
						__snake_case        =						max(output_differences.values()						) if output_differences else 0.0
						__snake_case        =						max(hidden_differences.values()						)
						if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
											raise ValueError(
											    """The cross-loaded TensorFlow model has different outputs, something went wrong!\n"""
											    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
											    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]						)
											    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
											    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]						)						)

						# Save the weights in a TF format (if needed) and confirms that the results are still good
						__snake_case        =						os.path.join(self._local_dir   ,     lowercase						)
						__snake_case        =						os.path.join(self._local_dir   ,     lowercase						)
						if (not os.path.exists(lowercase						) and not os.path.exists(lowercase						)) or self._new_weights:
											tf_from_pt_model.save_pretrained(self._local_dir						)
						del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

						__snake_case        =						tf_class.from_pretrained(self._local_dir						)
						__snake_case        =						tf_model(**lowercase   ,     output_hidden_states=lowercase						)

						__snake_case        =						self.find_pt_tf_differences(lowercase   ,     lowercase						)
						__snake_case        =						{k: v for k, v in conversion_differences.items() if """hidden""" not in k}
						__snake_case        =						{k: v for k, v in conversion_differences.items() if """hidden""" in k}
						if len(lowercase						) == 0 and architectures is not None:
											raise ValueError(
											    f'''Something went wrong -- the config file has architectures ({architectures}), but no model head'''
											    """ output was found. All outputs start with 'hidden'"""						)
						__snake_case        =						max(output_differences.values()						) if output_differences else 0.0
						__snake_case        =						max(hidden_differences.values()						)
						if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
											raise ValueError(
											    """The converted TensorFlow model has different outputs, something went wrong!\n"""
											    + f'''\nList of maximum output differences above the threshold ({self._max_error}):\n'''
											    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in output_differences.items() if v > self._max_error]						)
											    + f'''\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n'''
											    + """\n""".join([f'''{k}: {v:.3e}''' for k, v in hidden_differences.items() if v > self._max_error]						)						)

						__snake_case        =						"""Update TF weights""" if self._new_weights else """Add TF weights"""
						if self._push:
											repo.git_add(auto_lfs_track=lowercase						)
											repo.git_commit(lowercase						)
											repo.git_push(blocking=lowercase						)  # this prints a progress bar with the upload
											self._logger.warning(f'''TF weights pushed into {self._model_name}'''						)
						elif not self._no_pr:
											self._logger.warning("""Uploading the weights into a new PR..."""						)
											__snake_case        =						(
											    """Model converted by the [`transformers`' `pt_to_tf`"""
											    """ CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). """
											    """All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"""
											    f'''Maximum crossload output difference={max_crossload_output_diff:.3e}; '''
											    f'''Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n'''
											    f'''Maximum conversion output difference={max_conversion_output_diff:.3e}; '''
											    f'''Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n'''
											)
											if self._max_error > MAX_ERROR:
																commit_descrition += (
																    f'''\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!'''
																)
											if self._extra_commit_description:
																commit_descrition += "\n\n" + self._extra_commit_description

											# sharded model -> adds all related files (index and .h5 shards)
											if os.path.exists(lowercase						):
																__snake_case        =						[
																    CommitOperationAdd(path_in_repo=lowercase   ,     path_or_fileobj=lowercase						)
																]
																for shard_path in tf.io.gfile.glob(self._local_dir + """/tf_model-*.h5"""						):
																					operations += [
																					    CommitOperationAdd(path_in_repo=os.path.basename(lowercase						)   ,     path_or_fileobj=lowercase						)
																					]
											else:
																__snake_case        =						[CommitOperationAdd(path_in_repo=lowercase   ,     path_or_fileobj=lowercase						)]

											__snake_case        =						create_commit(
											    repo_id=self._model_name   ,     operations=lowercase   ,     commit_message=lowercase   ,     commit_description=lowercase   ,     repo_type="""model"""   ,     create_pr=lowercase   ,     ).pr_url
											self._logger.warning(f'''PR open in {hub_pr_url}'''						)


