






import colorsys
import io

import cva
import matplotlib as mpl
import matplotlib.colors as mplc
import matplotlib.figure as mplfigure
import numpy as np
import torch
from matplotlib.backends.backend_agg import FigureCanvasAgg

from utils import img_tensorize


__snake_case     :     List[str]       =  10_00


class      __SCREAMING_SNAKE_CASE       :






						def __init__(						self  ,    _UpperCamelCase  ,    _UpperCamelCase=1.2  ,    _UpperCamelCase="g"  ,    _UpperCamelCase=0.5  ,    _UpperCamelCase="-"  ,    _UpperCamelCase="test_out.jpg"  ,    _UpperCamelCase=True  ,    _UpperCamelCase=False  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=0.7  ,    ):






												"""simple docstring"""
												if isinstance(_UpperCamelCase  ,    torch.Tensor ):
																		lowerCAmelCase__         =    img.numpy().astype('np.uint8' )
												if isinstance(_UpperCamelCase  ,    _UpperCamelCase ):
																		lowerCAmelCase__         =    img_tensorize(_UpperCamelCase )
												assert isinstance(_UpperCamelCase  ,    np.ndarray )

												lowerCAmelCase__    ,							lowerCAmelCase__         =    img.shape[1], img.shape[0]
												lowerCAmelCase__         =    mplfigure.Figure(frameon=_UpperCamelCase )
												lowerCAmelCase__         =    fig.get_dpi()
												lowerCAmelCase__         =    (width * scale + 1E-2) / dpi
												lowerCAmelCase__         =    (height * scale + 1E-2) / dpi
												fig.set_size_inches(_UpperCamelCase  ,    _UpperCamelCase )
												lowerCAmelCase__         =    fig.add_axes([0.0, 0.0, 1.0, 1.0] )
												ax.axis('off' )
												ax.set_xlim(0.0  ,    _UpperCamelCase )
												ax.set_ylim(_UpperCamelCase )

												lowerCAmelCase__         =    saveas
												lowerCAmelCase__         =    rgb
												lowerCAmelCase__         =    pynb
												lowerCAmelCase__         =    img
												lowerCAmelCase__         =    edgecolor
												lowerCAmelCase__         =    0.5
												lowerCAmelCase__         =    linestyle
												lowerCAmelCase__         =    int(np.sqrt(min(_UpperCamelCase  ,    _UpperCamelCase ) ) * scale // 3 )
												lowerCAmelCase__         =    width
												lowerCAmelCase__         =    height
												lowerCAmelCase__         =    scale
												lowerCAmelCase__         =    fig
												lowerCAmelCase__         =    ax
												lowerCAmelCase__         =    pad
												lowerCAmelCase__         =    idaobj
												lowerCAmelCase__         =    idaattr
												lowerCAmelCase__         =    FigureCanvasAgg(_UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=None ):






												"""simple docstring"""
												if color is None:
																		lowerCAmelCase__         =    self.edgecolor
												((lowerCAmelCase__)    ,							(lowerCAmelCase__)    ,							(lowerCAmelCase__)    ,							(lowerCAmelCase__))         =    box
												lowerCAmelCase__         =    xa - xa
												lowerCAmelCase__         =    ya - ya
												self.ax.add_patch(
												    mpl.patches.Rectangle(
												        (xa, ya)  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    fill=_UpperCamelCase  ,    edgecolor=_UpperCamelCase  ,    linewidth=self.font_size // 3  ,    alpha=self.alpha  ,    linestyle=self.linestyle  ,    ) )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None ):






												"""simple docstring"""
												if len(boxes.shape ) > 2:
																		lowerCAmelCase__         =    boxes[0]
												if len(obj_ids.shape ) > 1:
																		lowerCAmelCase__         =    obj_ids[0]
												if len(obj_scores.shape ) > 1:
																		lowerCAmelCase__         =    obj_scores[0]
												if len(attr_ids.shape ) > 1:
																		lowerCAmelCase__         =    attr_ids[0]
												if len(attr_scores.shape ) > 1:
																		lowerCAmelCase__         =    attr_scores[0]
												if isinstance(_UpperCamelCase  ,    torch.Tensor ):
																		lowerCAmelCase__         =    boxes.numpy()
												if isinstance(_UpperCamelCase  ,    _UpperCamelCase ):
																		lowerCAmelCase__         =    np.array(_UpperCamelCase )
												assert isinstance(_UpperCamelCase  ,    np.ndarray )
												lowerCAmelCase__         =    np.prod(boxes[:, 2:] - boxes[:, :2]  ,    axis=1 )
												lowerCAmelCase__         =    np.argsort(-areas ).tolist()
												lowerCAmelCase__         =    boxes[sorted_idxs] if boxes is not None else None
												lowerCAmelCase__         =    obj_ids[sorted_idxs] if obj_ids is not None else None
												lowerCAmelCase__         =    obj_scores[sorted_idxs] if obj_scores is not None else None
												lowerCAmelCase__         =    attr_ids[sorted_idxs] if attr_ids is not None else None
												lowerCAmelCase__         =    attr_scores[sorted_idxs] if attr_scores is not None else None

												lowerCAmelCase__         =    [self._random_color(maximum=1 ) for _ in range(len(_UpperCamelCase ) )]
												lowerCAmelCase__         =    [assigned_colors[idx] for idx in sorted_idxs]
												if obj_ids is not None:
																		lowerCAmelCase__         =    self._create_text_labels_attr(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )
																		for i in range(len(_UpperCamelCase ) ):
																								lowerCAmelCase__         =    assigned_colors[i]
																								self.add_box(boxes[i]  ,    _UpperCamelCase )
																								self.draw_labels(labels[i]  ,    boxes[i]  ,    _UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__    ,							lowerCAmelCase__    ,							lowerCAmelCase__    ,							lowerCAmelCase__         =    box
												lowerCAmelCase__         =    (xa, ya)
												lowerCAmelCase__         =    (ya - ya) * (xa - xa)
												lowerCAmelCase__         =    _SMALL_OBJ * self.scale
												if instance_area < small or ya - ya < 40 * self.scale:
																		if ya >= self.height - 5:
																								lowerCAmelCase__         =    (xa, ya)
																		else:
																								lowerCAmelCase__         =    (xa, ya)

												lowerCAmelCase__         =    (ya - ya) / np.sqrt(self.height * self.width )
												lowerCAmelCase__         =    self._change_color_brightness(_UpperCamelCase  ,    brightness_factor=0.7 )
												lowerCAmelCase__         =    np.clip((height_ratio - 0.02) / 0.08 + 1  ,    1.2  ,    2 )
												font_size *= 0.75 * self.font_size

												self.draw_text(
												    text=_UpperCamelCase  ,    position=_UpperCamelCase  ,    color=_UpperCamelCase  ,    )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase="g"  ,    _UpperCamelCase="left"  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    0
												lowerCAmelCase__         =    self.font_size
												lowerCAmelCase__         =    np.maximum(list(mplc.to_rgb(_UpperCamelCase ) )  ,    0.2 )
												lowerCAmelCase__         =    max(0.8  ,    np.max(_UpperCamelCase ) )
												lowerCAmelCase__         =    {
												    'facecolor': 'black',
												    'alpha': self.alpha,
												    'pad': self.pad,
												    'edgecolor': 'none',
												}
												lowerCAmelCase__    ,							lowerCAmelCase__         =    position
												self.ax.text(
												    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    size=font_size * self.scale  ,    family='sans-serif'  ,    bbox=_UpperCamelCase  ,    verticalalignment='top'  ,    horizontalalignment=_UpperCamelCase  ,    color=_UpperCamelCase  ,    zorder=10  ,    rotation=_UpperCamelCase  ,    )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase=None ):






												"""simple docstring"""
												if saveas is None:
																		lowerCAmelCase__         =    self.saveas
												if saveas.lower().endswith('.jpg' ) or saveas.lower().endswith('.png' ):
																		cva.imwrite(
																		    _UpperCamelCase  ,    self._get_buffer()[:, :, ::-1]  ,    )
												else:
																		self.fig.savefig(_UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    [self.idaobj[i] for i in classes]
												lowerCAmelCase__         =    [self.idaattr[i] for i in attr_classes]
												lowerCAmelCase__         =    [
												    F"{label} {score:.2f} {attr} {attr_score:.2f}"
												    for label, score, attr, attr_score in zip(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )
												]
												return labels






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    [self.idaobj[i] for i in classes]
												if scores is not None:
																		if labels is None:
																								lowerCAmelCase__         =    ['{:.0f}%'.format(s * 1_00 ) for s in scores]
																		else:
																								lowerCAmelCase__         =    ['{} {:.0f}%'.format(_UpperCamelCase  ,    s * 1_00 ) for li, s in zip(_UpperCamelCase  ,    _UpperCamelCase )]
												return labels






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase=2_55 ):






												"""simple docstring"""
												lowerCAmelCase__         =    np.random.randint(0  ,    len(_COLORS ) )
												lowerCAmelCase__         =    _COLORS[idx] * maximum
												if not self.rgb:
																		lowerCAmelCase__         =    ret[::-1]
												return ret






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												if not self.pynb:
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    self.canvas.print_to_buffer()
																		if (width, height) != (self.width, self.height):
																								lowerCAmelCase__         =    cva.resize(self.img  ,    (width, height) )
																		else:
																								lowerCAmelCase__         =    self.img
												else:
																		lowerCAmelCase__         =    io.BytesIO()  # works for cairo backend
																		self.canvas.print_rgba(_UpperCamelCase )
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    self.width, self.height
																		lowerCAmelCase__         =    buf.getvalue()
																		lowerCAmelCase__         =    self.img

												lowerCAmelCase__         =    np.frombuffer(_UpperCamelCase  ,    dtype='uint8' )
												lowerCAmelCase__         =    buffer.reshape(_UpperCamelCase  ,    _UpperCamelCase  ,    4 )
												lowerCAmelCase__    ,							lowerCAmelCase__         =    np.split(_UpperCamelCase  ,    [3]  ,    axis=2 )

												try:
																		import numexpr as ne  # fuse them with numexpr

																		lowerCAmelCase__         =    ne.evaluate('img * (1 - alpha / 255.0) + rgb * (alpha / 255.0)' )
												except ImportError:
																		lowerCAmelCase__         =    alpha.astype('float32' ) / 2_55.0
																		lowerCAmelCase__         =    img * (1 - alpha) + rgb * alpha

												return visualized_image.astype('uint8' )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												assert brightness_factor >= -1.0 and brightness_factor <= 1.0
												lowerCAmelCase__         =    mplc.to_rgb(_UpperCamelCase )
												lowerCAmelCase__         =    colorsys.rgb_to_hls(*mplc.to_rgb(_UpperCamelCase ) )
												lowerCAmelCase__         =    polygon_color[1] + (brightness_factor * polygon_color[1])
												lowerCAmelCase__         =    0.0 if modified_lightness < 0.0 else modified_lightness
												lowerCAmelCase__         =    1.0 if modified_lightness > 1.0 else modified_lightness
												lowerCAmelCase__         =    colorsys.hls_to_rgb(polygon_color[0]  ,    _UpperCamelCase  ,    polygon_color[2] )
												return modified_color




__snake_case     :     Dict       =  (
    np.array(
        [
            0.000,
            0.447,
            0.741,
            0.850,
            0.325,
            0.098,
            0.929,
            0.694,
            0.125,
            0.494,
            0.184,
            0.556,
            0.466,
            0.674,
            0.188,
            0.301,
            0.745,
            0.933,
            0.635,
            0.078,
            0.184,
            0.300,
            0.300,
            0.300,
            0.600,
            0.600,
            0.600,
            1.000,
            0.000,
            0.000,
            1.000,
            0.500,
            0.000,
            0.749,
            0.749,
            0.000,
            0.000,
            1.000,
            0.000,
            0.000,
            0.000,
            1.000,
            0.667,
            0.000,
            1.000,
            0.333,
            0.333,
            0.000,
            0.333,
            0.667,
            0.000,
            0.333,
            1.000,
            0.000,
            0.667,
            0.333,
            0.000,
            0.667,
            0.667,
            0.000,
            0.667,
            1.000,
            0.000,
            1.000,
            0.333,
            0.000,
            1.000,
            0.667,
            0.000,
            1.000,
            1.000,
            0.000,
            0.000,
            0.333,
            0.500,
            0.000,
            0.667,
            0.500,
            0.000,
            1.000,
            0.500,
            0.333,
            0.000,
            0.500,
            0.333,
            0.333,
            0.500,
            0.333,
            0.667,
            0.500,
            0.333,
            1.000,
            0.500,
            0.667,
            0.000,
            0.500,
            0.667,
            0.333,
            0.500,
            0.667,
            0.667,
            0.500,
            0.667,
            1.000,
            0.500,
            1.000,
            0.000,
            0.500,
            1.000,
            0.333,
            0.500,
            1.000,
            0.667,
            0.500,
            1.000,
            1.000,
            0.500,
            0.000,
            0.333,
            1.000,
            0.000,
            0.667,
            1.000,
            0.000,
            1.000,
            1.000,
            0.333,
            0.000,
            1.000,
            0.333,
            0.333,
            1.000,
            0.333,
            0.667,
            1.000,
            0.333,
            1.000,
            1.000,
            0.667,
            0.000,
            1.000,
            0.667,
            0.333,
            1.000,
            0.667,
            0.667,
            1.000,
            0.667,
            1.000,
            1.000,
            1.000,
            0.000,
            1.000,
            1.000,
            0.333,
            1.000,
            1.000,
            0.667,
            1.000,
            0.333,
            0.000,
            0.000,
            0.500,
            0.000,
            0.000,
            0.667,
            0.000,
            0.000,
            0.833,
            0.000,
            0.000,
            1.000,
            0.000,
            0.000,
            0.000,
            0.167,
            0.000,
            0.000,
            0.333,
            0.000,
            0.000,
            0.500,
            0.000,
            0.000,
            0.667,
            0.000,
            0.000,
            0.833,
            0.000,
            0.000,
            1.000,
            0.000,
            0.000,
            0.000,
            0.167,
            0.000,
            0.000,
            0.333,
            0.000,
            0.000,
            0.500,
            0.000,
            0.000,
            0.667,
            0.000,
            0.000,
            0.833,
            0.000,
            0.000,
            1.000,
            0.000,
            0.000,
            0.000,
            0.143,
            0.143,
            0.143,
            0.857,
            0.857,
            0.857,
            1.000,
            1.000,
            1.000,
        ]
    )
    .astype(np.floataa)
    .reshape(-1, 3)
)













import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       PreTrainedTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Any				=       False
						_SCREAMING_SNAKE_CASE						:							Optional[int]				=       True
						_SCREAMING_SNAKE_CASE						:							int				=       '''tokenizer_file'''






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    False  # because we don't have pretrained_vocab_files_map
												super().setUp()
												lowerCAmelCase__         =    True

												lowerCAmelCase__         =    ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
												lowerCAmelCase__         =    'SaulLu/dummy-tokenizer-bytelevel-bpe'

												# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
												lowerCAmelCase__         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
												tokenizer.save_pretrained(self.tmpdirname )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer_with_special_tokens_change()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

												lowerCAmelCase__         =    ('a' for _ in range(10_00 ))
												lowerCAmelCase__         =    tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase  ,    length=10_00  ,    vocab_size=50 )

												lowerCAmelCase__         =    new_tokenizer.encode('a🤗' )
												self.assertEqual(_UpperCamelCase  ,    [64, 1_72, 2_53, 97, 2_45] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												lowerCAmelCase__         =    len(_UpperCamelCase )

												with tempfile.TemporaryDirectory() as tmp_dir:
																		# Hack to save this in the tokenizer_config.json
																		lowerCAmelCase__         =    ['tokenizer.4.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		json.dump(_UpperCamelCase  ,    open(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    'w' ) )

																		# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) + 1 )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

																		# Will need to be adjusted if we reach v42 and this test is still here.
																		# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																		shutil.move(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    os.path.join(_UpperCamelCase  ,    'tokenizer.42.0.0.json' ) )
																		lowerCAmelCase__         =    ['tokenizer.42.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
												lowerCAmelCase__         =    'hf-internal-testing/test-two-tokenizers'

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_97 )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												lowerCAmelCase__         =    '3.0.0'
												lowerCAmelCase__         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_96 )
												lowerCAmelCase__         =    json.loads(old_tokenizer._tokenizer.to_str() )
												self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# See https://github.com/huggingface/transformers/pull/12550
												# and https://github.com/huggingface/tokenizers/issues/537
												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel' )
												lowerCAmelCase__         =    'The Matrix is a 1999 science fiction action film.'

												with concurrent.futures.ThreadPoolExecutor() as executor:
																		lowerCAmelCase__         =    [executor.submit(self.fetch  ,    _UpperCamelCase  ,    _UpperCamelCase ) for i in range(10 )]
																		lowerCAmelCase__         =    [future.result() for future in futures]
																		self.assertEqual(_UpperCamelCase  ,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												return tokenizer.encode(_UpperCamelCase  ,    truncation='longest_first'  ,    padding='longest' )





