






import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer


class      __SCREAMING_SNAKE_CASE       ( tf.keras.layers.Layer):






						def __init__(						self  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase = None  ,    _UpperCamelCase = "longest"  ,    _UpperCamelCase = True  ,    _UpperCamelCase = 5_12  ,    _UpperCamelCase = None  ,    _UpperCamelCase = True  ,    _UpperCamelCase = True  ,    _UpperCamelCase = True  ,    **_UpperCamelCase  ,    ):






												"""simple docstring"""
												super().__init__()
												if use_fast_bert_tokenizer:
																		lowerCAmelCase__         =    FastBertTokenizer(
																		    _UpperCamelCase  ,    token_out_type=tf.intaa  ,    lower_case_nfd_strip_accents=_UpperCamelCase  ,    **_UpperCamelCase )
												else:
																		lowerCAmelCase__         =    tf.lookup.StaticVocabularyTable(
																		    tf.lookup.KeyValueTensorInitializer(
																		        keys=_UpperCamelCase  ,    key_dtype=tf.string  ,    values=tf.range(tf.size(_UpperCamelCase  ,    out_type=tf.intaa )  ,    dtype=tf.intaa )  ,    value_dtype=tf.intaa  ,    )  ,    num_oov_buckets=1  ,    )
																		lowerCAmelCase__         =    BertTokenizerLayer(
																		    _UpperCamelCase  ,    token_out_type=tf.intaa  ,    lower_case=_UpperCamelCase  ,    **_UpperCamelCase )

												lowerCAmelCase__         =    vocab_list
												lowerCAmelCase__         =    do_lower_case
												lowerCAmelCase__         =    cls_token_id or vocab_list.index('[CLS]' )
												lowerCAmelCase__         =    sep_token_id or vocab_list.index('[SEP]' )
												lowerCAmelCase__         =    pad_token_id or vocab_list.index('[PAD]' )
												lowerCAmelCase__         =    ShrinkLongestTrimmer(max_length - 3  ,    axis=1 )  # Allow room for special tokens
												lowerCAmelCase__         =    max_length
												lowerCAmelCase__         =    padding
												lowerCAmelCase__         =    truncation
												lowerCAmelCase__         =    pad_to_multiple_of
												lowerCAmelCase__         =    return_token_type_ids
												lowerCAmelCase__         =    return_attention_mask






						@classmethod
						def    UpperCamelCase__   (						cls  ,    _UpperCamelCase  ,    **_UpperCamelCase ):  # noqa: F821






												"""simple docstring"""
												lowerCAmelCase__         =    kwargs.pop('do_lower_case'  ,    _UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
												lowerCAmelCase__         =    kwargs.pop('cls_token_id'  ,    _UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
												lowerCAmelCase__         =    kwargs.pop('sep_token_id'  ,    _UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
												lowerCAmelCase__         =    kwargs.pop('pad_token_id'  ,    _UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

												lowerCAmelCase__         =    tokenizer.get_vocab()
												lowerCAmelCase__         =    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]  ,    key=lambda _UpperCamelCase : x[1] )
												lowerCAmelCase__         =    [entry[0] for entry in vocab]
												return cls(
												    vocab_list=_UpperCamelCase  ,    do_lower_case=_UpperCamelCase  ,    cls_token_id=_UpperCamelCase  ,    sep_token_id=_UpperCamelCase  ,    pad_token_id=_UpperCamelCase  ,    **_UpperCamelCase  ,    )






						@classmethod
						def    UpperCamelCase__   (						cls  ,    _UpperCamelCase  ,    *_UpperCamelCase  ,    **_UpperCamelCase ):






												"""simple docstring"""
												try:
																		lowerCAmelCase__         =    BertTokenizer.from_pretrained(_UpperCamelCase  ,    *_UpperCamelCase  ,    **_UpperCamelCase )
												except:  # noqa: E722
																		from .tokenization_bert_fast import BertTokenizerFast

																		lowerCAmelCase__         =    BertTokenizerFast.from_pretrained(_UpperCamelCase  ,    *_UpperCamelCase  ,    **_UpperCamelCase )
												return cls.from_tokenizer(_UpperCamelCase  ,    **_UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if self.do_lower_case:
																		lowerCAmelCase__         =    case_fold_utfa(_UpperCamelCase )
												lowerCAmelCase__         =    self.tf_tokenizer.tokenize(_UpperCamelCase )
												return tokens.merge_dims(1  ,    -1 )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    ):






												"""simple docstring"""
												if padding is None:
																		lowerCAmelCase__         =    self.padding
												if padding not in ("longest", "max_length"):
																		raise ValueError('Padding must be either \'longest\' or \'max_length\'!' )
												if max_length is not None and text_pair is not None:
																		# Because we have to instantiate a Trimmer to do it properly
																		raise ValueError('max_length cannot be overridden at call time when truncating paired texts!' )
												if max_length is None:
																		lowerCAmelCase__         =    self.max_length
												if truncation is None:
																		lowerCAmelCase__         =    self.truncation
												if pad_to_multiple_of is None:
																		lowerCAmelCase__         =    self.pad_to_multiple_of
												if return_token_type_ids is None:
																		lowerCAmelCase__         =    self.return_token_type_ids
												if return_attention_mask is None:
																		lowerCAmelCase__         =    self.return_attention_mask
												if not isinstance(_UpperCamelCase  ,    tf.Tensor ):
																		lowerCAmelCase__         =    tf.convert_to_tensor(_UpperCamelCase )
												if text_pair is not None and not isinstance(_UpperCamelCase  ,    tf.Tensor ):
																		lowerCAmelCase__         =    tf.convert_to_tensor(_UpperCamelCase )
												if text_pair is not None:
																		if text.shape.rank > 1:
																								raise ValueError('text argument should not be multidimensional when a text pair is supplied!' )
																		if text_pair.shape.rank > 1:
																								raise ValueError('text_pair should not be multidimensional!' )
												if text.shape.rank == 2:
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    text[:, 0], text[:, 1]
												lowerCAmelCase__         =    self.unpaired_tokenize(_UpperCamelCase )
												if text_pair is None:  # Unpaired text
																		if truncation:
																								lowerCAmelCase__         =    text[:, : max_length - 2]  # Allow room for special tokens
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    combine_segments(
																		    (text,)  ,    start_of_sequence_id=self.cls_token_id  ,    end_of_segment_id=self.sep_token_id )
												else:  # Paired text
																		lowerCAmelCase__         =    self.unpaired_tokenize(_UpperCamelCase )
																		if truncation:
																								lowerCAmelCase__    ,							lowerCAmelCase__         =    self.paired_trimmer.trim([text, text_pair] )
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    combine_segments(
																		    (text, text_pair)  ,    start_of_sequence_id=self.cls_token_id  ,    end_of_segment_id=self.sep_token_id )
												if padding == "longest":
																		lowerCAmelCase__         =    input_ids.bounding_shape(axis=1 )
																		if pad_to_multiple_of is not None:
																								# No ceiling division in tensorflow, so we negate floordiv instead
																								lowerCAmelCase__         =    pad_to_multiple_of * (-tf.math.floordiv(-pad_length  ,    _UpperCamelCase ))
												else:
																		lowerCAmelCase__         =    max_length

												lowerCAmelCase__    ,							lowerCAmelCase__         =    pad_model_inputs(_UpperCamelCase  ,    max_seq_length=_UpperCamelCase  ,    pad_value=self.pad_token_id )
												lowerCAmelCase__         =    {'input_ids': input_ids}
												if return_attention_mask:
																		lowerCAmelCase__         =    attention_mask
												if return_token_type_ids:
																		lowerCAmelCase__    ,							lowerCAmelCase__         =    pad_model_inputs(
																		    _UpperCamelCase  ,    max_seq_length=_UpperCamelCase  ,    pad_value=self.pad_token_id )
																		lowerCAmelCase__         =    token_type_ids
												return output







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												return {
												    "vocab_list": self.vocab_list,
												    "do_lower_case": self.do_lower_case,
												    "cls_token_id": self.cls_token_id,
												    "sep_token_id": self.sep_token_id,
												    "pad_token_id": self.pad_token_id,
												}













import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       PreTrainedTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Any				=       False
						_SCREAMING_SNAKE_CASE						:							Optional[int]				=       True
						_SCREAMING_SNAKE_CASE						:							int				=       '''tokenizer_file'''






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    False  # because we don't have pretrained_vocab_files_map
												super().setUp()
												lowerCAmelCase__         =    True

												lowerCAmelCase__         =    ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
												lowerCAmelCase__         =    'SaulLu/dummy-tokenizer-bytelevel-bpe'

												# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
												lowerCAmelCase__         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
												tokenizer.save_pretrained(self.tmpdirname )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer_with_special_tokens_change()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

												lowerCAmelCase__         =    ('a' for _ in range(10_00 ))
												lowerCAmelCase__         =    tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase  ,    length=10_00  ,    vocab_size=50 )

												lowerCAmelCase__         =    new_tokenizer.encode('a🤗' )
												self.assertEqual(_UpperCamelCase  ,    [64, 1_72, 2_53, 97, 2_45] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												lowerCAmelCase__         =    len(_UpperCamelCase )

												with tempfile.TemporaryDirectory() as tmp_dir:
																		# Hack to save this in the tokenizer_config.json
																		lowerCAmelCase__         =    ['tokenizer.4.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		json.dump(_UpperCamelCase  ,    open(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    'w' ) )

																		# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) + 1 )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

																		# Will need to be adjusted if we reach v42 and this test is still here.
																		# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																		shutil.move(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    os.path.join(_UpperCamelCase  ,    'tokenizer.42.0.0.json' ) )
																		lowerCAmelCase__         =    ['tokenizer.42.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
												lowerCAmelCase__         =    'hf-internal-testing/test-two-tokenizers'

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_97 )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												lowerCAmelCase__         =    '3.0.0'
												lowerCAmelCase__         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_96 )
												lowerCAmelCase__         =    json.loads(old_tokenizer._tokenizer.to_str() )
												self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# See https://github.com/huggingface/transformers/pull/12550
												# and https://github.com/huggingface/tokenizers/issues/537
												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel' )
												lowerCAmelCase__         =    'The Matrix is a 1999 science fiction action film.'

												with concurrent.futures.ThreadPoolExecutor() as executor:
																		lowerCAmelCase__         =    [executor.submit(self.fetch  ,    _UpperCamelCase  ,    _UpperCamelCase ) for i in range(10 )]
																		lowerCAmelCase__         =    [future.result() for future in futures]
																		self.assertEqual(_UpperCamelCase  ,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												return tokenizer.encode(_UpperCamelCase  ,    truncation='longest_first'  ,    padding='longest' )





