


"""simple docstring"""
from manim import *





class   __A  (snake_case__):





							'''simple docstring'''

							def 		lowerCAmelCase							(					self				:   Optional[Any]							)       ->Dict:



														"""simple docstring"""



														snake_case_     							=		Rectangle(height=0.5		,							width=0.5							)
														snake_case_     							=		Rectangle(height=0.46		,							width=0.46							).set_stroke(width=0							)
														snake_case_     							=		Rectangle(height=0.25		,							width=0.25							)

														snake_case_     							=		[mem.copy() for i in range(6							)]
														snake_case_     							=		[mem.copy() for i in range(6							)]
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		VGroup(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		Text("""CPU"""		,							font_size=24							)
														snake_case_     							=		Group(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0.5		,							aligned_edge=UpperCAmelCase_							)
														cpu.move_to([-2.5, -0.5, 0]							)
														self.add(UpperCAmelCase_							)

														snake_case_     							=		[mem.copy() for i in range(4							)]
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		Text("""GPU"""		,							font_size=24							)
														snake_case_     							=		Group(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0.5		,							aligned_edge=UpperCAmelCase_							)
														gpu.move_to([-1, -1, 0]							)
														self.add(UpperCAmelCase_							)

														snake_case_     							=		[mem.copy() for i in range(6							)]
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)

														snake_case_     							=		Text("""Model"""		,							font_size=24							)
														snake_case_     							=		Group(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0.5		,							aligned_edge=UpperCAmelCase_							)
														model.move_to([3, -1.0, 0]							)
														self.add(UpperCAmelCase_							)

														snake_case_     							=		[]
														snake_case_     							=		[]

														for i, rect in enumerate(UpperCAmelCase_							):
																					rect.set_stroke(UpperCAmelCase_							)

																					snake_case_     							=		Rectangle(height=0.46 / 4		,							width=0.46 / 3							).set_stroke(width=0.0							).set_fill(UpperCAmelCase_		,							opacity=0.7							)

																					if i == 0:
																												cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)		,							buff=0.02		,							direction=UpperCAmelCase_							)
																												cpu_target.set_x(cpu_target.get_x() + 0.1							)
																					elif i == 3:
																												cpu_target.next_to(model_cpu_arr[0]		,							direction=UpperCAmelCase_		,							buff=0.0							)
																					else:
																												cpu_target.next_to(model_cpu_arr[i - 1]		,							direction=UpperCAmelCase_		,							buff=0.0							)
																					self.add(UpperCAmelCase_							)
																					model_cpu_arr.append(UpperCAmelCase_							)

														self.add(*UpperCAmelCase_		,							*UpperCAmelCase_							)

														snake_case_     							=		[meta_mem.copy() for i in range(6							)]
														snake_case_     							=		[meta_mem.copy() for i in range(6							)]
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		VGroup(*UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		VGroup(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0							)
														snake_case_     							=		Text("""Disk"""		,							font_size=24							)
														snake_case_     							=		Group(UpperCAmelCase_		,							UpperCAmelCase_							).arrange(UpperCAmelCase_		,							buff=0.5		,							aligned_edge=UpperCAmelCase_							)
														disk.move_to([-4.0, -1.25, 0]							)
														self.add(UpperCAmelCase_		,							UpperCAmelCase_							)

														snake_case_     							=		[]

														for i in range(6							):
																					snake_case_     							=		fill.copy().set_fill(UpperCAmelCase_		,							opacity=0.8							)
																					target.move_to(disk_left_col_base[i]							).scale(0.5							)
																					cpu_disk_arr.append(UpperCAmelCase_							)

														self.add(*UpperCAmelCase_							)

														snake_case_     							=		Square(side_length=2.2							)
														key.move_to([-5, 2, 0]							)

														snake_case_     							=		MarkupText(
														    F"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""		,							font_size=18		,							)

														key_text.move_to([-5, 2.4, 0]							)

														self.add(UpperCAmelCase_		,							UpperCAmelCase_							)

														snake_case_     							=		MarkupText(
														    F"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""		,							font_size=18		,							)

														blue_text.next_to(UpperCAmelCase_		,							DOWN * 2.4		,							aligned_edge=key_text.get_left()							)
														self.add(UpperCAmelCase_							)

														snake_case_     							=		MarkupText(F"""The offloaded weights are all sent to the CPU."""		,							font_size=24							)
														step_a.move_to([2, 2, 0]							)

														self.play(Write(UpperCAmelCase_		,							run_time=3							)							)

														for i in range(6							):
																					snake_case_     							=		cpu_disk_arr[i]
																					snake_case_     							=		rect.copy().set_fill(UpperCAmelCase_		,							opacity=0.8							).scale(2.0							)
																					cpa.generate_target()
																					cpa.target.move_to(model_base[i]							)

																					if i == 0:
																												rect.set_fill(UpperCAmelCase_		,							opacity=0.8							)
																												rect.generate_target()
																												rect.target.move_to(cpu_left_col_base[0]							).scale(2.0							)

																												self.remove(
																												    *UpperCAmelCase_		,							*UpperCAmelCase_		,							)

																					else:
																												rect.generate_target()
																												rect.target.move_to(cpu_left_col_base[i]							).scale(2.0							)
																					self.play(MoveToTarget(UpperCAmelCase_							)		,							MoveToTarget(UpperCAmelCase_							)		,							model_base[i].animate.set_stroke(UpperCAmelCase_							)							)
														self.play(FadeOut(UpperCAmelCase_							)							)

														snake_case_     							=		MarkupText(
														    F"""Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed."""		,							font_size=24		,							)
														step_a.move_to([2, 2, 0]							)

														self.play(Write(UpperCAmelCase_		,							run_time=3							)							)

														snake_case_     							=		[]
														snake_case_     							=		[]
														for i in range(6							):
																					snake_case_     							=		Arrow(start=UpperCAmelCase_		,							end=UpperCAmelCase_		,							color=UpperCAmelCase_		,							buff=0.5							)
																					a.next_to(model_base[i].get_left()		,							UpperCAmelCase_		,							buff=0.2							)
																					arrows.append(UpperCAmelCase_							)
																					animations.append(Write(UpperCAmelCase_							)							)
														self.play(*UpperCAmelCase_							)
														self.wait()







"""simple docstring"""
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class   __A  (snake_case__    ,		snake_case__    ,		snake_case__):





							'''simple docstring'''

							@register_to_config
							def __init__(					self				:   List[str]		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   float		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   str		,							UpperCAmelCase_				:   bool = False		,							)       ->Dict:



														"""simple docstring"""



														super().__init__()

														snake_case_     							=		nn.Linear(UpperCAmelCase_		,							UpperCAmelCase_		,							bias=UpperCAmelCase_							)

														snake_case_     							=		nn.Embedding(UpperCAmelCase_		,							UpperCAmelCase_							)
														snake_case_     							=		False

														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)

														snake_case_     							=		TaConfig(
														    d_model=UpperCAmelCase_		,							num_heads=UpperCAmelCase_		,							d_kv=UpperCAmelCase_		,							d_ff=UpperCAmelCase_		,							feed_forward_proj=UpperCAmelCase_		,							dropout_rate=UpperCAmelCase_		,							is_decoder=UpperCAmelCase_		,							is_encoder_decoder=UpperCAmelCase_		,							)
														snake_case_     							=		nn.ModuleList()
														for lyr_num in range(UpperCAmelCase_							):
																					snake_case_     							=		TaBlock(UpperCAmelCase_							)
																					self.encoders.append(UpperCAmelCase_							)

														snake_case_     							=		TaLayerNorm(UpperCAmelCase_							)
														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)



							def 		lowerCAmelCase							(					self				:   Optional[Any]		,							UpperCAmelCase_				:   Any		,							UpperCAmelCase_				:   List[str]							)       ->List[Any]:



														"""simple docstring"""



														snake_case_     							=		self.input_proj(UpperCAmelCase_							)

														# terminal relative positional encodings
														snake_case_     							=		encoder_inputs.shape[1]
														snake_case_     							=		torch.arange(UpperCAmelCase_		,							device=encoder_inputs.device							)

														snake_case_     							=		encoder_inputs_mask.sum(-1							)
														snake_case_     							=		torch.roll(input_positions.unsqueeze(0							)		,							tuple(seq_lens.tolist()							)		,							dims=0							)
														x += self.position_encoding(UpperCAmelCase_							)

														snake_case_     							=		self.dropout_pre(UpperCAmelCase_							)

														# inverted the attention mask
														snake_case_     							=		encoder_inputs.size()
														snake_case_     							=		self.get_extended_attention_mask(UpperCAmelCase_		,							UpperCAmelCase_							)

														for lyr in self.encoders:
																					snake_case_     							=		lyr(UpperCAmelCase_		,							UpperCAmelCase_							)[0]
														snake_case_     							=		self.layer_norm(UpperCAmelCase_							)

														return self.dropout_post(UpperCAmelCase_							), encoder_inputs_mask



