




import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__lowerCAmelCase						:							List[Any]			     =						16
__lowerCAmelCase						:							List[Any]			     =						32




def      __magic_name__							(					A						:   Union[str, Any],    A						:   str  ):






			'''simple docstring'''







			if args.with_tracking:
						a       =     Accelerator(
						    cpu=args.cpu,    mixed_precision=args.mixed_precision,    log_with="all",    project_dir=args.project_dir  )
			else:
						a       =     Accelerator(cpu=args.cpu,    mixed_precision=args.mixed_precision  )

			if hasattr(args.checkpointing_steps,    "isdigit"  ):
						if args.checkpointing_steps == "epoch":
									a       =     args.checkpointing_steps
						elif args.checkpointing_steps.isdigit():
									a       =     int(args.checkpointing_steps  )
						else:
									raise ValueError(
									    F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""  )
			else:
						a       =     None
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			a       =     config["lr"]
			a       =     int(config["num_epochs"]  )
			a       =     int(config["seed"]  )
			a       =     int(config["batch_size"]  )

			# We need to initialize the trackers we use, and also store our configuration
			if args.with_tracking:
						a       =     os.path.split(A  )[-1].split("."  )[0]
						accelerator.init_trackers(A,    A  )

			a       =     AutoTokenizer.from_pretrained("bert-base-cased"  )
			a       =     load_dataset("glue",    "mrpc"  )
			a       =     evaluate.load("glue",    "mrpc"  )

			def tokenize_function(A						:   Any  ):
						# max_length=None => use the model max length (it's actually the default)
						a       =     tokenizer(examples["sentence1"],    examples["sentence2"],    truncation=A,    max_length=A  )
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						a       =     datasets.map(
						    A,    batched=A,    remove_columns=["idx", "sentence1", "sentence2"],    )

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			a       =     tokenized_datasets.rename_column("label",    "labels"  )

			# If the batch size is too big we use gradient accumulation
			a       =     1
			if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
						a       =     batch_size // MAX_GPU_BATCH_SIZE
						a       =     MAX_GPU_BATCH_SIZE

			def collate_fn(A						:   Optional[int]  ):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						a       =     128 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									a       =     16
						elif accelerator.mixed_precision != "no":
									a       =     8
						else:
									a       =     None

						return tokenizer.pad(
						    A,    padding="longest",    max_length=A,    pad_to_multiple_of=A,    return_tensors="pt",    )

			# Instantiate dataloaders.
			a       =     DataLoader(
			    tokenized_datasets["train"],    shuffle=A,    collate_fn=A,    batch_size=A  )
			a       =     DataLoader(
			    tokenized_datasets["validation"],    shuffle=A,    collate_fn=A,    batch_size=A  )

			set_seed(A  )

			# Instantiate the model (we build the model here so that the seed also control new weights initialization)
			a       =     AutoModelForSequenceClassification.from_pretrained("bert-base-cased",    return_dict=A  )

			# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
			# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
			# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
			a       =     model.to(accelerator.device  )

			# Instantiate optimizer
			a       =     AdamW(params=model.parameters(),    lr=A  )

			# Instantiate scheduler
			a       =     get_linear_schedule_with_warmup(
			    optimizer=A,    num_warmup_steps=100,    num_training_steps=(len(A  ) * num_epochs) // gradient_accumulation_steps,    )

			# Prepare everything
			# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
			# prepare method.
			a       ,						a       ,						a       ,						a       ,						a       =     accelerator.prepare(
			    A,    A,    A,    A,    A  )

			# We need to keep track of how many total steps we have iterated over
			a       =     0
			# We also need to keep track of the stating epoch so files are named properly
			a       =     0

			# Potentially load in the weights and states from a previous save
			if args.resume_from_checkpoint:
						if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
									accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""  )
									accelerator.load_state(args.resume_from_checkpoint  )
									a       =     os.path.basename(args.resume_from_checkpoint  )
						else:
									# Get the most recent checkpoint
									a       =     [f.name for f in os.scandir(os.getcwd()  ) if f.is_dir()]
									dirs.sort(key=os.path.getctime  )
									a       =     dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
						# Extract `epoch_{i}` or `step_{i}`
						a       =     os.path.splitext(A  )[0]

						if "epoch" in training_difference:
									a       =     int(training_difference.replace("epoch_",    ""  )  ) + 1
									a       =     None
						else:
									a       =     int(training_difference.replace("step_",    ""  )  )
									a       =     resume_step // len(A  )
									resume_step -= starting_epoch * len(A  )

    # Now we train the model
			for epoch in range(A,    A  ):
						model.train()
						if args.with_tracking:
									a       =     0
						if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
									# We need to skip steps until we reach the resumed step
									a       =     accelerator.skip_first_batches(A,    A  )
									overall_step += resume_step
						else:
									# After the first iteration though, we need to go back to the original dataloader
									a       =     train_dataloader
						for step, batch in enumerate(A  ):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device  )
									a       =     model(**A  )
									a       =     outputs.loss
									a       =     loss / gradient_accumulation_steps
									# We keep track of the loss at each epoch
									if args.with_tracking:
												total_loss += loss.detach().float()
									accelerator.backward(A  )
									if step % gradient_accumulation_steps == 0:
												optimizer.step()
												lr_scheduler.step()
												optimizer.zero_grad()

									overall_step += 1

									if isinstance(A,    A  ):
												a       =     F"""step_{overall_step}"""
												if overall_step % checkpointing_steps == 0:
															if args.output_dir is not None:
																		a       =     os.path.join(args.output_dir,    A  )
															accelerator.save_state(A  )

						model.eval()
						for step, batch in enumerate(A  ):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device  )
									with torch.no_grad():
												a       =     model(**A  )
									a       =     outputs.logits.argmax(dim=-1  )
									a       ,						a       =     accelerator.gather_for_metrics((predictions, batch["labels"])  )
									metric.add_batch(
									    predictions=A,    references=A,    )

						a       =     metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(F"""epoch {epoch}:""",    A  )
						if args.with_tracking:
									accelerator.log(
									    {
									        "accuracy": eval_metric["accuracy"],
									        "f1": eval_metric["f1"],
									        "train_loss": total_loss.item() / len(A  ),
									        "epoch": epoch,
									    },    step=A,    )

						if checkpointing_steps == "epoch":
									a       =     F"""epoch_{epoch}"""
									if args.output_dir is not None:
												a       =     os.path.join(args.output_dir,    A  )
									accelerator.save_state(A  )

			if args.with_tracking:
						accelerator.end_training()




def      __magic_name__							(					):






			'''simple docstring'''







			a       =     argparse.ArgumentParser(description="Simple example of training script."  )
			parser.add_argument(
			    "--mixed_precision",    type=A,    default=A,    choices=["no", "fp16", "bf16", "fp8"],    help="Whether to use mixed precision. Choose"
			    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
			    "and an Nvidia Ampere GPU.",    )
			parser.add_argument("--cpu",    action="store_true",    help="If passed, will train on the CPU."  )
			parser.add_argument(
			    "--checkpointing_steps",    type=A,    default=A,    help="Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.",    )
			parser.add_argument(
			    "--resume_from_checkpoint",    type=A,    default=A,    help="If the training should continue from a checkpoint folder.",    )
			parser.add_argument(
			    "--with_tracking",    action="store_true",    help="Whether to load in all available experiment trackers from the environment and use them for logging.",    )
			parser.add_argument(
			    "--output_dir",    type=A,    default=".",    help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory.",    )
			parser.add_argument(
			    "--project_dir",    type=A,    default="logs",    help="Location on where to store experiment tracking logs` and relevent project information",    )
			a       =     parser.parse_args()
			a       =     {"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
			training_function(A,    A  )


if __name__ == "__main__":
		main()












from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXLMForMultipleChoice,
		    TFXLMForQuestionAnsweringSimple,
		    TFXLMForSequenceClassification,
		    TFXLMForTokenClassification,
		    TFXLMModel,
		    TFXLMWithLMHeadModel,
		    XLMConfig,
		)




class        snake_case__				:
				"""simple docstring"""



				def __init__( self							:	Optional[Any]   , __lowerCamelCase							:	Any   , )   ->	Any:
							a       =     parent
							a       =     13
							a       =     7
							a       =     True
							a       =     True
							a       =     True
							a       =     True
							a       =     True
							a       =     False
							a       =     False
							a       =     False
							a       =     2
							a       =     99
							a       =     0
							a       =     32
							a       =     2
							a       =     4
							a       =     0.1
							a       =     0.1
							a       =     5_12
							a       =     16
							a       =     2
							a       =     0.02
							a       =     3
							a       =     4
							a       =     "last"
							a       =     True
							a       =     None
							a       =     0





				def 				__UpperCAmelCase				( self							:	int				)   ->	Tuple:
							a       =     ids_tensor([self.batch_size, self.seq_length]   , self.vocab_size				)
							a       =     random_attention_mask([self.batch_size, self.seq_length]   , dtype=tf.floataa				)

							a       =     None
							if self.use_input_lengths:
										a       =     (
										    ids_tensor([self.batch_size]   , vocab_size=2				) + self.seq_length - 2
										)  # small variation of seq_length

							a       =     None
							if self.use_token_type_ids:
										a       =     ids_tensor([self.batch_size, self.seq_length]   , self.n_langs				)

							a       =     None
							a       =     None
							a       =     None
							if self.use_labels:
										a       =     ids_tensor([self.batch_size]   , self.type_sequence_label_size				)
										a       =     ids_tensor([self.batch_size, self.seq_length]   , self.num_labels				)
										a       =     ids_tensor([self.batch_size]   , 2   , dtype=tf.floataa				)
										a       =     ids_tensor([self.batch_size]   , self.num_choices				)

							a       =     XLMConfig(
							    vocab_size=self.vocab_size   , n_special=self.n_special   , emb_dim=self.hidden_size   , n_layers=self.num_hidden_layers   , n_heads=self.num_attention_heads   , dropout=self.hidden_dropout_prob   , attention_dropout=self.attention_probs_dropout_prob   , gelu_activation=self.gelu_activation   , sinusoidal_embeddings=self.sinusoidal_embeddings   , asm=self.asm   , causal=self.causal   , n_langs=self.n_langs   , max_position_embeddings=self.max_position_embeddings   , initializer_range=self.initializer_range   , summary_type=self.summary_type   , use_proj=self.use_proj   , bos_token_id=self.bos_token_id   , )

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_lengths,
							    sequence_labels,
							    token_labels,
							    is_impossible_labels,
							    choice_labels,
							    input_mask,
							)





				def 				__UpperCAmelCase				( self							:	List[str]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Any   , )   ->	Union[str, Any]:
							a       =     TFXLMModel(config=__lowerCamelCase				)
							a       =     {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
							a       =     model(__lowerCamelCase				)

							a       =     [input_ids, input_mask]
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size)				)





				def 				__UpperCAmelCase				( self							:	Union[str, Any]   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Any   , )   ->	Optional[Any]:
							a       =     TFXLMWithLMHeadModel(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
							a       =     model(__lowerCamelCase				)

							a       =     outputs

							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.vocab_size)				)





				def 				__UpperCAmelCase				( self							:	Tuple   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	List[Any]   , )   ->	int:
							a       =     TFXLMForQuestionAnsweringSimple(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths}

							a       =     model(__lowerCamelCase				)

							self.parent.assertEqual(result.start_logits.shape   , (self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape   , (self.batch_size, self.seq_length)				)





				def 				__UpperCAmelCase				( self							:	Dict   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	int   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	Optional[int]   , )   ->	Optional[int]:
							a       =     TFXLMForSequenceClassification(__lowerCamelCase				)

							a       =     {"input_ids": input_ids, "lengths": input_lengths}

							a       =     model(__lowerCamelCase				)

							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.type_sequence_label_size)				)





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , )   ->	Optional[Any]:
							a       =     self.num_labels
							a       =     TFXLMForTokenClassification(config=__lowerCamelCase				)
							a       =     {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.num_labels)				)





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Union[str, Any]   , __lowerCamelCase							:	Dict   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	Optional[Any]   , __lowerCamelCase							:	str   , __lowerCamelCase							:	List[str]   , __lowerCamelCase							:	Dict   , )   ->	List[Any]:
							a       =     self.num_choices
							a       =     TFXLMForMultipleChoice(config=__lowerCamelCase				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     tf.tile(tf.expand_dims(__lowerCamelCase   , 1				)   , (1, self.num_choices, 1)				)
							a       =     {
							    "input_ids": multiple_choice_inputs_ids,
							    "attention_mask": multiple_choice_input_mask,
							    "token_type_ids": multiple_choice_token_type_ids,
							}
							a       =     model(__lowerCamelCase				)
							self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.num_choices)				)





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	str:
							a       =     self.prepare_config_and_inputs()
							(
							    (
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						(
							    a
							)       ,						
							)       =     config_and_inputs
							a       =     {
							    "input_ids": input_ids,
							    "token_type_ids": token_type_ids,
							    "langs": token_type_ids,
							    "lengths": input_lengths,
							}
							return config, inputs_dict




@require_tf
class        snake_case__				(_UpperCamelCase       ,			_UpperCamelCase       ,			unittest.TestCase  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				int									=   (
				    (
				        TFXLMModel,
				        TFXLMWithLMHeadModel,
				        TFXLMForSequenceClassification,
				        TFXLMForQuestionAnsweringSimple,
				        TFXLMForTokenClassification,
				        TFXLMForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE_			:				List[str]									=   (
				    (TFXLMWithLMHeadModel,) if is_tf_available() else ()
				)  # TODO (PVP): Check other models whether language generation is also applicable
				SCREAMING_SNAKE_CASE_			:				List[Any]									=   (
				    {
				        """feature-extraction""": TFXLMModel,
				        """fill-mask""": TFXLMWithLMHeadModel,
				        """question-answering""": TFXLMForQuestionAnsweringSimple,
				        """text-classification""": TFXLMForSequenceClassification,
				        """text-generation""": TFXLMWithLMHeadModel,
				        """token-classification""": TFXLMForTokenClassification,
				        """zero-shot""": TFXLMForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				SCREAMING_SNAKE_CASE_			:				Tuple									=   False
				SCREAMING_SNAKE_CASE_			:				Dict									=   False





				def 				__UpperCAmelCase				( self							:	Any   , __lowerCamelCase							:	Tuple   , __lowerCamelCase							:	List[Any]   , __lowerCamelCase							:	Any   , __lowerCamelCase							:	Optional[int]   , __lowerCamelCase							:	List[Any]				)   ->	Dict:
							if (
							    pipeline_test_casse_name == "QAPipelineTests"
							    and tokenizer_name is not None
							    and not tokenizer_name.endswith("Fast"				)
							):
										# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
										# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
										# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
										return True

							return False





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	Tuple:
							a       =     TFXLMModelTester(self				)
							a       =     ConfigTester(self   , config_class=__lowerCamelCase   , emb_dim=37				)





				def 				__UpperCAmelCase				( self							:	List[str]				)   ->	Optional[int]:
							self.config_tester.run_common_tests()





				def 				__UpperCAmelCase				( self							:	Dict				)   ->	List[Any]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_model(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	Dict:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_lm_head(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Dict:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_qa(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	Optional[int]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_sequence_classif(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	Optional[Any]:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_for_token_classification(*__lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	str:
							a       =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_xlm_for_multiple_choice(*__lowerCamelCase				)





				@slow
				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	List[Any]:
							for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										a       =     TFXLMModel.from_pretrained(__lowerCamelCase				)
										self.assertIsNotNone(__lowerCamelCase				)




@require_tf
class        snake_case__				(unittest.TestCase  ):
				"""simple docstring"""



				@slow
				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	str:
							a       =     TFXLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"				)
							a       =     tf.convert_to_tensor([[14, 4_47]]   , dtype=tf.intaa				)  # the president
							a       =     [
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							    14,
							    4_47,
							]  # the president the president the president the president the president the president the president the president the president the president
							# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
							a       =     model.generate(__lowerCamelCase   , do_sample=__lowerCamelCase				)
							self.assertListEqual(output_ids[0].numpy().tolist()   , __lowerCamelCase				)






