






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece


class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    list(tokenizer_class.max_model_input_sizes.keys() )
												for model_name in sa_models[:1]:
																		lowerCAmelCase__         =    tokenizer_class.from_pretrained(_UpperCamelCase )
																		self.assertIsNotNone(_UpperCamelCase )
																		self.assertIsInstance(_UpperCamelCase  ,    _UpperCamelCase )
																		self.assertIsInstance(_UpperCamelCase  ,    _UpperCamelCase )

																		for special_tok in tokenizer.all_special_tokens:
																								self.assertIsInstance(_UpperCamelCase  ,    _UpperCamelCase )
																								lowerCAmelCase__         =    tokenizer.convert_tokens_to_ids(_UpperCamelCase )
																								self.assertIsInstance(_UpperCamelCase  ,    _UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase = None ):






												"""simple docstring"""
												lowerCAmelCase__         =    pickle.dumps(_UpperCamelCase )
												self.assertIsNotNone(_UpperCamelCase )

												lowerCAmelCase__         =    pickle.loads(_UpperCamelCase )

												# Ensure is_fast is correctly restored
												self.assertEqual(be_restored.is_fast  ,    be_original.is_fast )

												# Ensure encodings are potentially correctly restored
												if be_original.is_fast:
																		self.assertIsNotNone(be_restored.encodings )
												else:
																		self.assertIsNone(be_restored.encodings )

												# Ensure the keys are the same
												for original_v, restored_v in zip(be_original.values()  ,    be_restored.values() ):
																		if equal_op:
																								self.assertTrue(equal_op(_UpperCamelCase  ,    _UpperCamelCase ) )
																		else:
																								self.assertEqual(_UpperCamelCase  ,    _UpperCamelCase )






						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												self.check_tokenizer_from_pretrained(_UpperCamelCase )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												self.assertEqual(TensorType('tf' )  ,    TensorType.TENSORFLOW )
												self.assertEqual(TensorType('pt' )  ,    TensorType.PYTORCH )
												self.assertEqual(TensorType('np' )  ,    TensorType.NUMPY )






						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												import numpy as np

												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    BertTokenizerFast.from_pretrained('bert-base-cased' )

												# Python no tensor
												with self.subTest('BatchEncoding (Python, return_tensors=None)' ):
																		self.assert_dump_and_restore(tokenizer_p('Small example to encode' ) )

												with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_p('Small example to encode'  ,    return_tensors=TensorType.NUMPY )  ,    np.array_equal )

												with self.subTest('BatchEncoding (Rust, return_tensors=None)' ):
																		self.assert_dump_and_restore(tokenizer_r('Small example to encode' ) )

												with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_r('Small example to encode'  ,    return_tensors=TensorType.NUMPY )  ,    np.array_equal )






						@require_tf
						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												import tensorflow as tf

												def tf_array_equals(_UpperCamelCase  ,    _UpperCamelCase ):
																		return tf.reduce_all(tf.equal(_UpperCamelCase  ,    _UpperCamelCase ) )

												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    BertTokenizerFast.from_pretrained('bert-base-cased' )

												with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_p('Small example to encode'  ,    return_tensors=TensorType.TENSORFLOW )  ,    _UpperCamelCase )

												with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_r('Small example to encode'  ,    return_tensors=TensorType.TENSORFLOW )  ,    _UpperCamelCase )






						@require_torch
						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												import torch

												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    BertTokenizerFast.from_pretrained('bert-base-cased' )

												with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_p('Small example to encode'  ,    return_tensors=TensorType.PYTORCH )  ,    torch.equal )

												with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)' ):
																		self.assert_dump_and_restore(
																		    tokenizer_r('Small example to encode'  ,    return_tensors=TensorType.PYTORCH )  ,    torch.equal )






						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    BertTokenizerFast.from_pretrained('bert-base-cased' )

												with self.subTest('Python Tokenizer' ):
																		self.assertFalse(tokenizer_p('Small example to_encode' ).is_fast )

												with self.subTest('Rust Tokenizer' ):
																		self.assertTrue(tokenizer_r('Small example to_encode' ).is_fast )






						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BertTokenizerFast.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    tokenizer_r(['Test', '\xad', 'test']  ,    is_split_into_words=_UpperCamelCase )

												self.assertEqual(encoded.word_to_tokens(0 )  ,    TokenSpan(start=1  ,    end=2 ) )
												self.assertEqual(encoded.word_to_tokens(1 )  ,    _UpperCamelCase )
												self.assertEqual(encoded.word_to_tokens(2 )  ,    TokenSpan(start=2  ,    end=3 ) )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='np' )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (2, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (2,) )
												# test converting the converted
												with CaptureStderr() as cs:
																		lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='np' )
												self.assertFalse(len(cs.err )  ,    msg=F"should have no warning, but got {cs.err}" )

												lowerCAmelCase__         =    BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='np'  ,    prepend_batch_axis=_UpperCamelCase )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (1, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (1,) )






						@require_torch
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='pt' )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (2, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (2,) )
												# test converting the converted
												with CaptureStderr() as cs:
																		lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='pt' )
												self.assertFalse(len(cs.err )  ,    msg=F"should have no warning, but got {cs.err}" )

												lowerCAmelCase__         =    BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='pt'  ,    prepend_batch_axis=_UpperCamelCase )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (1, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (1,) )






						@require_tf
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='tf' )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (2, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (2,) )
												# test converting the converted
												with CaptureStderr() as cs:
																		lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='tf' )
												self.assertFalse(len(cs.err )  ,    msg=F"should have no warning, but got {cs.err}" )

												lowerCAmelCase__         =    BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='tf'  ,    prepend_batch_axis=_UpperCamelCase )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (1, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (1,) )






						@require_flax
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='jax' )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (2, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (2,) )
												# test converting the converted
												with CaptureStderr() as cs:
																		lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='jax' )
												self.assertFalse(len(cs.err )  ,    msg=F"should have no warning, but got {cs.err}" )

												lowerCAmelCase__         =    BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
												lowerCAmelCase__         =    batch.convert_to_tensors(tensor_type='jax'  ,    prepend_batch_axis=_UpperCamelCase )
												self.assertEqual(tensor_batch['inputs'].shape  ,    (1, 3) )
												self.assertEqual(tensor_batch['labels'].shape  ,    (1,) )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    [{'input_ids': np.array([0, 1, 2] )}, {'input_ids': np.array([0, 1, 2, 3] )}]
												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )

												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase )
												self.assertTrue(isinstance(batch['input_ids']  ,    np.ndarray ) )
												self.assertEqual(batch['input_ids'].tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase  ,    return_tensors='np' )
												self.assertTrue(isinstance(batch['input_ids']  ,    np.ndarray ) )
												self.assertEqual(batch['input_ids'].tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






						@require_torch
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												import torch

												lowerCAmelCase__         =    [{'input_ids': torch.tensor([0, 1, 2] )}, {'input_ids': torch.tensor([0, 1, 2, 3] )}]
												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )

												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase )
												self.assertTrue(isinstance(batch['input_ids']  ,    torch.Tensor ) )
												self.assertEqual(batch['input_ids'].tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase  ,    return_tensors='pt' )
												self.assertTrue(isinstance(batch['input_ids']  ,    torch.Tensor ) )
												self.assertEqual(batch['input_ids'].tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






						@require_tf
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												import tensorflow as tf

												lowerCAmelCase__         =    [{'input_ids': tf.constant([0, 1, 2] )}, {'input_ids': tf.constant([0, 1, 2, 3] )}]
												lowerCAmelCase__         =    BertTokenizer.from_pretrained('bert-base-cased' )

												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase )
												self.assertTrue(isinstance(batch['input_ids']  ,    tf.Tensor ) )
												self.assertEqual(batch['input_ids'].numpy().tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
												lowerCAmelCase__         =    tokenizer.pad(_UpperCamelCase  ,    padding=_UpperCamelCase  ,    return_tensors='tf' )
												self.assertTrue(isinstance(batch['input_ids']  ,    tf.Tensor ) )
												self.assertEqual(batch['input_ids'].numpy().tolist()  ,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )






						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    Tokenizer(WordPiece(unk_token='[UNK]' ) )
												PreTrainedTokenizerFast(tokenizer_object=_UpperCamelCase )







						@require_tokenizers
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    Tokenizer(WordPiece(unk_token='[UNK]' ) )
												with tempfile.TemporaryDirectory() as tmpdirname:
																		bert_tokenizer.save(os.path.join(_UpperCamelCase  ,    'tokenizer.json' ) )
																		PreTrainedTokenizerFast(tokenizer_file=os.path.join(_UpperCamelCase  ,    'tokenizer.json' ) )













import numpy as np


class      __SCREAMING_SNAKE_CASE       :






						def __init__(						self  ,    _UpperCamelCase=5  ,    _UpperCamelCase=5 ):






												"""simple docstring"""
												lowerCAmelCase__         =    depth
												lowerCAmelCase__         =    0
												lowerCAmelCase__         =    None
												lowerCAmelCase__         =    None
												lowerCAmelCase__         =    min_leaf_size
												lowerCAmelCase__         =    None






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												if labels.ndim != 1:
																		print('Error: Input labels must be one dimensional' )

												return np.mean((labels - prediction) ** 2 )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												if x.ndim != 1:
																		print('Error: Input data set must be one dimensional' )
																		return
												if len(_UpperCamelCase ) != len(_UpperCamelCase ):
																		print('Error: X and y have different lengths' )
																		return
												if y.ndim != 1:
																		print('Error: Data set labels must be one dimensional' )
																		return

												if len(_UpperCamelCase ) < 2 * self.min_leaf_size:
																		lowerCAmelCase__         =    np.mean(_UpperCamelCase )
																		return

												if self.depth == 1:
																		lowerCAmelCase__         =    np.mean(_UpperCamelCase )
																		return

												lowerCAmelCase__         =    0
												lowerCAmelCase__         =    self.mean_squared_error(_UpperCamelCase  ,    np.mean(_UpperCamelCase ) ) * 2
												for i in range(len(_UpperCamelCase ) ):
																		if len(x[:i] ) < self.min_leaf_size:
																								continue
																		elif len(x[i:] ) < self.min_leaf_size:
																								continue
																		else:
																								lowerCAmelCase__         =    self.mean_squared_error(x[:i]  ,    np.mean(y[:i] ) )
																								lowerCAmelCase__         =    self.mean_squared_error(x[i:]  ,    np.mean(y[i:] ) )
																								lowerCAmelCase__         =    error_left + error_right
																								if error < min_error:
																														lowerCAmelCase__         =    i
																														lowerCAmelCase__         =    error

												if best_split != 0:
																		lowerCAmelCase__         =    x[:best_split]
																		lowerCAmelCase__         =    y[:best_split]
																		lowerCAmelCase__         =    x[best_split:]
																		lowerCAmelCase__         =    y[best_split:]

																		lowerCAmelCase__         =    x[best_split]
																		lowerCAmelCase__         =    DecisionTree(
																		    depth=self.depth - 1  ,    min_leaf_size=self.min_leaf_size )
																		lowerCAmelCase__         =    DecisionTree(
																		    depth=self.depth - 1  ,    min_leaf_size=self.min_leaf_size )
																		self.left.train(_UpperCamelCase  ,    _UpperCamelCase )
																		self.right.train(_UpperCamelCase  ,    _UpperCamelCase )
												else:
																		lowerCAmelCase__         =    np.mean(_UpperCamelCase )

												return







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if self.prediction is not None:
																		return self.prediction
												elif self.left or self.right is not None:
																		if x >= self.decision_boundary:
																								return self.right.predict(_UpperCamelCase )
																		else:
																								return self.left.predict(_UpperCamelCase )
												else:
																		print('Error: Decision tree not yet trained' )
																		return None




class      __SCREAMING_SNAKE_CASE       :






						@staticmethod
						def    UpperCamelCase__   (						_UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    float(0 )
												for label in labels:
																		squared_error_sum += (label - prediction) ** 2

												return float(squared_error_sum / labels.size )







def 						_UpperCamelCase (     )	-> List[Any]:

						"""simple docstring"""



						lowerCAmelCase__         =    np.arange(-1.0     ,							1.0     ,							0.005		)
						lowerCAmelCase__         =    np.sin(UpperCamelCase_		)

						lowerCAmelCase__         =    DecisionTree(depth=10     ,							min_leaf_size=10		)
						tree.train(UpperCamelCase_     ,							UpperCamelCase_		)

						lowerCAmelCase__         =    (np.random.rand(10		) * 2) - 1
						lowerCAmelCase__         =    np.array([tree.predict(UpperCamelCase_		) for x in test_cases]		)
						lowerCAmelCase__         =    np.mean((predictions - test_cases) ** 2		)

						print('Test values: ' + str(UpperCamelCase_		)		)
						print('Predictions: ' + str(UpperCamelCase_		)		)
						print('Average error: ' + str(UpperCamelCase_		)		)


if __name__ == "__main__":
					main()
					import doctest

					doctest.testmod(name="""mean_squarred_error""", verbose=True)





