


"""simple docstring"""







from manim import *
class 				lowerCamelCase							(      __SCREAMING_SNAKE_CASE			):


							"""simple docstring"""






							def        lowerCAmelCase  (    self			:  Union[str, Any]					):
													UpperCAmelCase__												=     Rectangle(height=0.5		,			width=0.5					)
													UpperCAmelCase__												=     Rectangle(height=0.46		,			width=0.46					).set_stroke(width=0					)
													UpperCAmelCase__												=     Rectangle(height=0.25		,			width=0.25					)

													UpperCAmelCase__												=     [mem.copy() for i in range(6					)]
													UpperCAmelCase__												=     [mem.copy() for i in range(6					)]
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     VGroup(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     Text('''CPU'''		,			font_size=24					)
													UpperCAmelCase__												=     Group(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0.5		,			aligned_edge=lowerCamelCase_					)
													cpu.move_to([-2.5, -0.5, 0]					)
													self.add(lowerCamelCase_					)

													UpperCAmelCase__												=     [mem.copy() for i in range(4					)]
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     Text('''GPU'''		,			font_size=24					)
													UpperCAmelCase__												=     Group(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0.5		,			aligned_edge=lowerCamelCase_					)
													gpu.move_to([-1, -1, 0]					)
													self.add(lowerCamelCase_					)

													UpperCAmelCase__												=     [mem.copy() for i in range(6					)]
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)

													UpperCAmelCase__												=     Text('''Model'''		,			font_size=24					)
													UpperCAmelCase__												=     Group(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0.5		,			aligned_edge=lowerCamelCase_					)
													model.move_to([3, -1.0, 0]					)
													self.add(lowerCamelCase_					)

													UpperCAmelCase__												=     []
													UpperCAmelCase__												=     []

													for i, rect in enumerate(lowerCamelCase_					):
																			rect.set_stroke(lowerCamelCase_					)

																			UpperCAmelCase__												=     Rectangle(height=0.46 / 4		,			width=0.46 / 3					).set_stroke(width=0.0					).set_fill(lowerCamelCase_		,			opacity=0.7					)

																			if i == 0:
																									cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)		,			buff=0.02		,			direction=lowerCamelCase_					)
																									cpu_target.set_x(cpu_target.get_x() + 0.1					)
																			elif i == 3:
																									cpu_target.next_to(model_cpu_arr[0]		,			direction=lowerCamelCase_		,			buff=0.0					)
																			else:
																									cpu_target.next_to(model_cpu_arr[i - 1]		,			direction=lowerCamelCase_		,			buff=0.0					)
																			self.add(lowerCamelCase_					)
																			model_cpu_arr.append(lowerCamelCase_					)

													self.add(*lowerCamelCase_		,			*lowerCamelCase_					)

													UpperCAmelCase__												=     [meta_mem.copy() for i in range(6					)]
													UpperCAmelCase__												=     [meta_mem.copy() for i in range(6					)]
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     VGroup(*lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     VGroup(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0					)
													UpperCAmelCase__												=     Text('''Disk'''		,			font_size=24					)
													UpperCAmelCase__												=     Group(lowerCamelCase_		,			lowerCamelCase_					).arrange(lowerCamelCase_		,			buff=0.5		,			aligned_edge=lowerCamelCase_					)
													disk.move_to([-4.0, -1.25, 0]					)
													self.add(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     []

													for i in range(6					):
																			UpperCAmelCase__												=     fill.copy().set_fill(lowerCamelCase_		,			opacity=0.8					)
																			target.move_to(disk_left_col_base[i]					).scale(0.5					)
																			cpu_disk_arr.append(lowerCamelCase_					)

													self.add(*lowerCamelCase_					)

													UpperCAmelCase__												=     Square(side_length=2.2					)
													key.move_to([-5, 2, 0]					)

													UpperCAmelCase__												=     MarkupText(
													    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""		,			font_size=18		,			)

													key_text.move_to([-5, 2.4, 0]					)

													self.add(lowerCamelCase_		,			lowerCamelCase_					)

													UpperCAmelCase__												=     MarkupText(
													    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""		,			font_size=18		,			)

													blue_text.next_to(lowerCamelCase_		,			DOWN * 2.4		,			aligned_edge=key_text.get_left()					)
													self.add(lowerCamelCase_					)

													UpperCAmelCase__												=     MarkupText(f"""The offloaded weights are all sent to the CPU."""		,			font_size=24					)
													step_a.move_to([2, 2, 0]					)

													self.play(Write(lowerCamelCase_		,			run_time=3					)					)

													for i in range(6					):
																			UpperCAmelCase__												=     cpu_disk_arr[i]
																			UpperCAmelCase__												=     rect.copy().set_fill(lowerCamelCase_		,			opacity=0.8					).scale(2.0					)
																			cpa.generate_target()
																			cpa.target.move_to(model_base[i]					)

																			if i == 0:
																									rect.set_fill(lowerCamelCase_		,			opacity=0.8					)
																									rect.generate_target()
																									rect.target.move_to(cpu_left_col_base[0]					).scale(2.0					)

																									self.remove(
																									    *lowerCamelCase_		,			*lowerCamelCase_		,			)

																			else:
																									rect.generate_target()
																									rect.target.move_to(cpu_left_col_base[i]					).scale(2.0					)
																			self.play(MoveToTarget(lowerCamelCase_					)		,			MoveToTarget(lowerCamelCase_					)		,			model_base[i].animate.set_stroke(lowerCamelCase_					)					)
													self.play(FadeOut(lowerCamelCase_					)					)

													UpperCAmelCase__												=     MarkupText(
													    f"""Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed."""		,			font_size=24		,			)
													step_a.move_to([2, 2, 0]					)

													self.play(Write(lowerCamelCase_		,			run_time=3					)					)

													UpperCAmelCase__												=     []
													UpperCAmelCase__												=     []
													for i in range(6					):
																			UpperCAmelCase__												=     Arrow(start=lowerCamelCase_		,			end=lowerCamelCase_		,			color=lowerCamelCase_		,			buff=0.5					)
																			a.next_to(model_base[i].get_left()		,			lowerCamelCase_		,			buff=0.2					)
																			arrows.append(lowerCamelCase_					)
																			animations.append(Write(lowerCamelCase_					)					)
													self.play(*lowerCamelCase_					)
													self.wait()



"""simple docstring"""







import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp






def        a__			(							):





						"""simple docstring"""

						UpperCAmelCase__												=     ArgumentParser(
						    description=(
						        '''PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'''
						    )			)

						# Optional arguments for the launch helper
						parser.add_argument('''--num_cores'''						,      type=_UpperCamelCase						,      default=1						,      help='''Number of TPU cores to use (1 or 8).'''			)

						# positional
						parser.add_argument(
						    '''training_script'''						,      type=_UpperCamelCase						,      help=(
						        '''The full path to the single TPU training '''
						        '''program/script to be launched in parallel, '''
						        '''followed by all the arguments for the '''
						        '''training script'''
						    )						,      )

						# rest from the training program
						parser.add_argument('''training_script_args'''						,      nargs=_UpperCamelCase			)

						return parser.parse_args()






def        a__			(							):





						"""simple docstring"""

						UpperCAmelCase__												=     parse_args()

						# Import training_script as a module.
						UpperCAmelCase__												=     Path(args.training_script			)
						sys.path.append(str(script_fpath.parent.resolve()			)			)
						UpperCAmelCase__												=     script_fpath.stem
						UpperCAmelCase__												=     importlib.import_module(_UpperCamelCase			)

						# Patch sys.argv
						UpperCAmelCase__												=     [args.training_script] + args.training_script_args + ['''--tpu_num_cores''', str(args.num_cores			)]

						xmp.spawn(mod._mp_fn						,      args=()						,      nprocs=args.num_cores			)


if __name__ == "__main__":
						main()