





from manim import *

class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase  ):


      def    a (self       :      List[str]      ):





         """simple docstring"""



         __snake_case							       =   Rectangle(height=0.5				,			width=0.5      )
         __snake_case							       =   Rectangle(height=0.2_5				,			width=0.2_5      )
         __snake_case							       =   Rectangle(height=0.4_6				,			width=0.4_6      ).set_stroke(width=0      )

         __snake_case							       =   [mem.copy() for i in range(6      )]
         __snake_case							       =   [mem.copy() for i in range(6      )]
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   VGroup(a__				,			a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   Text('''CPU'''				,			font_size=24      )
         __snake_case							       =   Group(a__				,			a__      ).arrange(a__				,			buff=0.5				,			aligned_edge=a__      )
         cpu.move_to([-2.5, -0.5, 0]      )
         self.add(a__      )

         __snake_case							       =   [mem.copy() for i in range(4      )]
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   Text('''GPU'''				,			font_size=24      )
         __snake_case							       =   Group(a__				,			a__      ).arrange(a__				,			buff=0.5				,			aligned_edge=a__      )
         gpu.move_to([-1, -1, 0]      )
         self.add(a__      )

         __snake_case							       =   [mem.copy() for i in range(6      )]
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )

         __snake_case							       =   Text('''Model'''				,			font_size=24      )
         __snake_case							       =   Group(a__				,			a__      ).arrange(a__				,			buff=0.5				,			aligned_edge=a__      )
         model.move_to([3, -1.0, 0]      )
         self.add(a__      )

         __snake_case							       =   []
         __snake_case							       =   []
         __snake_case							       =   []

         for i, rect in enumerate(a__      ):
            rect.set_stroke(a__      )

            __snake_case							       =   Rectangle(height=0.4_6 / 4				,			width=0.4_6 / 3      ).set_stroke(width=0.0      ).set_fill(a__				,			opacity=0.7      )

            if i == 0:
               cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      )				,			buff=0.0_2				,			direction=a__      )
               cpu_target.set_x(cpu_target.get_x() + 0.1      )
            elif i == 3:
               cpu_target.next_to(model_cpu_arr[0]				,			direction=a__				,			buff=0.0      )
            else:
               cpu_target.next_to(model_cpu_arr[i - 1]				,			direction=a__				,			buff=0.0      )
            self.add(a__      )
            model_cpu_arr.append(a__      )

         self.add(*a__				,			*a__				,			*a__      )

         __snake_case							       =   [mem.copy() for i in range(6      )]
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )

         __snake_case							       =   Text('''Loaded Checkpoint'''				,			font_size=24      )
         __snake_case							       =   Group(a__				,			a__      ).arrange(a__				,			buff=0.5				,			aligned_edge=a__      )
         checkpoint.move_to([3, 0.5, 0]      )

         self.add(a__      )

         __snake_case							       =   []
         __snake_case							       =   []

         for i, rect in enumerate(a__      ):
            __snake_case							       =   fill.copy().set_fill(a__				,			opacity=0.7      )
            target.move_to(a__      )
            ckpt_arr.append(a__      )

            __snake_case							       =   target.copy()
            if i < 5:
               cpu_target.move_to(cpu_left_col_base[i + 1]      )
            else:
               cpu_target.move_to(cpu_right_col_base[i - 5]      )
            ckpt_cpu_arr.append(a__      )
         self.add(*a__				,			*a__      )

         __snake_case							       =   Square(side_length=2.2      )
         key.move_to([-5, 2, 0]      )

         __snake_case							       =   MarkupText(
             f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""				,			font_size=18				,			)

         key_text.move_to([-5, 2.4, 0]      )

         self.add(a__				,			a__      )

         __snake_case							       =   MarkupText(
             f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""				,			font_size=18				,			)

         blue_text.next_to(a__				,			DOWN * 2.4				,			aligned_edge=key_text.get_left()      )
         self.add(a__      )

         __snake_case							       =   MarkupText(
             f"""Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."""				,			font_size=24				,			)
         step_a.move_to([2, 2, 0]      )

         __snake_case							       =   [meta_mem.copy() for i in range(6      )]
         __snake_case							       =   [meta_mem.copy() for i in range(6      )]
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   VGroup(*a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   VGroup(a__				,			a__      ).arrange(a__				,			buff=0      )
         __snake_case							       =   Text('''Disk'''				,			font_size=24      )
         __snake_case							       =   Group(a__				,			a__      ).arrange(a__				,			buff=0.5				,			aligned_edge=a__      )
         disk.move_to([-4.0, -1.2_5, 0]      )
         self.play(Write(a__				,			run_time=3      )				,			Write(a__				,			run_time=1      )				,			Create(a__				,			run_time=1      )      )

         __snake_case							       =   []
         for i, rect in enumerate(a__      ):
            __snake_case							       =   rect.copy()
            target.generate_target()
            target.target.move_to(disk_left_col_base[i]      ).scale(0.5      )
            animations.append(MoveToTarget(a__				,			run_time=1.5      )      )
         self.play(*a__      )

         self.play(FadeOut(a__      )      )

         __snake_case							       =   MarkupText(f"""Then, the checkpoint is removed from memory\nthrough garbage collection."""				,			font_size=24      )
         step_a.move_to([2, 2, 0]      )

         self.play(Write(a__				,			run_time=3      )      )

         self.play(
             FadeOut(a__				,			a__				,			*a__				,			*a__      )				,			)

         self.wait()








import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_				=			get_tests_dir('fixtures/test_sentencepiece_bpe.model')

class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				List[Any]							=    BartphoTokenizer
      A_						:				List[str]							=    False
      A_						:				Optional[Any]							=    True


      def    a (self       :      Tuple      ):





         """simple docstring"""



         super().setUp()

         __snake_case							       =   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
         __snake_case							       =   dict(zip(a__				,			range(len(a__      )      )      )      )
         __snake_case							       =   {'''unk_token''': '''<unk>'''}

         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''monolingual_vocab_file''']      )
         with open(self.monolingual_vocab_file				,			'''w'''				,			encoding='''utf-8'''      ) as fp:
            for token in vocab_tokens:
               fp.write(f"""{token} {vocab_tokens[token]}\n"""      )

         __snake_case							       =   BartphoTokenizer(a__				,			self.monolingual_vocab_file				,			**self.special_tokens_map      )
         tokenizer.save_pretrained(self.tmpdirname      )


      def    a (self       :      str				,			**a__       :      str      ):





         """simple docstring"""



         kwargs.update(self.special_tokens_map      )
         return BartphoTokenizer.from_pretrained(self.tmpdirname				,			**a__      )


      def    a (self       :      str				,			a__       :      Any      ):





         """simple docstring"""



         __snake_case							       =   '''This is a là test'''
         __snake_case							       =   '''This is a<unk><unk> test'''
         return input_text, output_text


      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   BartphoTokenizer(a__				,			self.monolingual_vocab_file				,			**self.special_tokens_map      )
         __snake_case							       =   '''This is a là test'''
         __snake_case							       =   '''▁This ▁is ▁a ▁l à ▁t est'''.split()
         __snake_case							       =   tokenizer.tokenize(a__      )
         self.assertListEqual(a__				,			a__      )

         __snake_case							       =   tokens + [tokenizer.unk_token]
         __snake_case							       =   [4, 5, 6, 3, 3, 7, 8, 3]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )				,			a__      )

