"""simple docstring"""






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece
class 							snake_case			(					unittest.TestCase							):




					"""simple docstring"""

					def 						__lowerCAmelCase		(							self  :      Tuple       ,lowerCamelCase__  :      Optional[int]     ):
									UpperCAmelCase__			 =				list(tokenizer_class.max_model_input_sizes.keys()     )
									for model_name in sa_models[:1]:
													UpperCAmelCase__			 =				tokenizer_class.from_pretrained(lowerCamelCase__     )
													self.assertIsNotNone(lowerCamelCase__     )
													self.assertIsInstance(lowerCamelCase__       ,lowerCamelCase__     )
													self.assertIsInstance(lowerCamelCase__       ,lowerCamelCase__     )

													for special_tok in tokenizer.all_special_tokens:
																	self.assertIsInstance(lowerCamelCase__       ,lowerCamelCase__     )
																	UpperCAmelCase__			 =				tokenizer.convert_tokens_to_ids(lowerCamelCase__     )
																	self.assertIsInstance(lowerCamelCase__       ,lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      Optional[Any]       ,lowerCamelCase__  :      BatchEncoding       ,lowerCamelCase__  :      Optional[Callable] = None     ):
									UpperCAmelCase__			 =				pickle.dumps(lowerCamelCase__     )
									self.assertIsNotNone(lowerCamelCase__     )

									UpperCAmelCase__			 =				pickle.loads(lowerCamelCase__     )

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast       ,be_original.is_fast     )

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings     )
									else:
													self.assertIsNone(be_restored.encodings     )

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()       ,be_restored.values()     ):
													if equal_op:
																	self.assertTrue(equal_op(lowerCamelCase__       ,lowerCamelCase__     )     )
													else:
																	self.assertEqual(lowerCamelCase__       ,lowerCamelCase__     )







					@slow
					def 						__lowerCAmelCase		(							self  :      Dict     ):
									self.check_tokenizer_from_pretrained(lowerCamelCase__     )







					def 						__lowerCAmelCase		(							self  :      Optional[Any]     ):
									self.assertEqual(TensorType('tf'     )       ,TensorType.TENSORFLOW     )
									self.assertEqual(TensorType('pt'     )       ,TensorType.PYTORCH     )
									self.assertEqual(TensorType('np'     )       ,TensorType.NUMPY     )







					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      int     ):
									import numpy as np

									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )
									UpperCAmelCase__			 =				BertTokenizerFast.from_pretrained('bert-base-cased'     )

									# Python no tensor
									with self.subTest('BatchEncoding (Python, return_tensors=None)'     ):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode'     )     )

									with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'     ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'       ,return_tensors=TensorType.NUMPY     )       ,np.array_equal     )

									with self.subTest('BatchEncoding (Rust, return_tensors=None)'     ):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode'     )     )

									with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'     ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'       ,return_tensors=TensorType.NUMPY     )       ,np.array_equal     )







					@require_tf
					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      Tuple     ):
									import tensorflow as tf

									def tf_array_equals(lowerCamelCase__  :      Tuple       ,lowerCamelCase__  :      List[Any]     ):
													return tf.reduce_all(tf.equal(lowerCamelCase__       ,lowerCamelCase__     )     )

									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )
									UpperCAmelCase__			 =				BertTokenizerFast.from_pretrained('bert-base-cased'     )

									with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'     ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'       ,return_tensors=TensorType.TENSORFLOW     )       ,lowerCamelCase__     )

									with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'     ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'       ,return_tensors=TensorType.TENSORFLOW     )       ,lowerCamelCase__     )







					@require_torch
					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      Optional[int]     ):
									import torch

									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )
									UpperCAmelCase__			 =				BertTokenizerFast.from_pretrained('bert-base-cased'     )

									with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'     ):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'       ,return_tensors=TensorType.PYTORCH     )       ,torch.equal     )

									with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'     ):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'       ,return_tensors=TensorType.PYTORCH     )       ,torch.equal     )







					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      Optional[int]     ):
									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )
									UpperCAmelCase__			 =				BertTokenizerFast.from_pretrained('bert-base-cased'     )

									with self.subTest('Python Tokenizer'     ):
													self.assertFalse(tokenizer_p('Small example to_encode'     ).is_fast     )

									with self.subTest('Rust Tokenizer'     ):
													self.assertTrue(tokenizer_r('Small example to_encode'     ).is_fast     )







					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      List[str]     ):
									UpperCAmelCase__			 =				BertTokenizerFast.from_pretrained('bert-base-cased'     )
									UpperCAmelCase__			 =				tokenizer_r(['Test', '\xad', 'test']       ,is_split_into_words=lowerCamelCase__     )

									self.assertEqual(encoded.word_to_tokens(0     )       ,TokenSpan(start=1       ,end=2     )     )
									self.assertEqual(encoded.word_to_tokens(1     )       ,lowerCamelCase__     )
									self.assertEqual(encoded.word_to_tokens(2     )       ,TokenSpan(start=2       ,end=3     )     )







					def 						__lowerCAmelCase		(							self  :      Optional[int]     ):
									UpperCAmelCase__			 =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='np'     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(2, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(2,)     )
									# test converting the converted
									with CaptureStderr() as cs:
													UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='np'     )
									self.assertFalse(len(cs.err     )       ,msg=f'''should have no warning, but got {cs.err}'''     )

									UpperCAmelCase__			 =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='np'       ,prepend_batch_axis=lowerCamelCase__     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(1, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(1,)     )







					@require_torch
					def 						__lowerCAmelCase		(							self  :      Tuple     ):
									UpperCAmelCase__			 =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='pt'     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(2, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(2,)     )
									# test converting the converted
									with CaptureStderr() as cs:
													UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='pt'     )
									self.assertFalse(len(cs.err     )       ,msg=f'''should have no warning, but got {cs.err}'''     )

									UpperCAmelCase__			 =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='pt'       ,prepend_batch_axis=lowerCamelCase__     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(1, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(1,)     )







					@require_tf
					def 						__lowerCAmelCase		(							self  :      List[Any]     ):
									UpperCAmelCase__			 =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='tf'     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(2, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(2,)     )
									# test converting the converted
									with CaptureStderr() as cs:
													UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='tf'     )
									self.assertFalse(len(cs.err     )       ,msg=f'''should have no warning, but got {cs.err}'''     )

									UpperCAmelCase__			 =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='tf'       ,prepend_batch_axis=lowerCamelCase__     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(1, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(1,)     )







					@require_flax
					def 						__lowerCAmelCase		(							self  :      Dict     ):
									UpperCAmelCase__			 =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='jax'     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(2, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(2,)     )
									# test converting the converted
									with CaptureStderr() as cs:
													UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='jax'     )
									self.assertFalse(len(cs.err     )       ,msg=f'''should have no warning, but got {cs.err}'''     )

									UpperCAmelCase__			 =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
									UpperCAmelCase__			 =				batch.convert_to_tensors(tensor_type='jax'       ,prepend_batch_axis=lowerCamelCase__     )
									self.assertEqual(tensor_batch['inputs'].shape       ,(1, 3)     )
									self.assertEqual(tensor_batch['labels'].shape       ,(1,)     )







					def 						__lowerCAmelCase		(							self  :      Optional[Any]     ):
									UpperCAmelCase__			 =				[{'input_ids': np.array([0, 1, 2]     )}, {'input_ids': np.array([0, 1, 2, 3]     )}]
									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )

									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__     )
									self.assertTrue(isinstance(batch['input_ids']       ,np.ndarray     )     )
									self.assertEqual(batch['input_ids'].tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__       ,return_tensors='np'     )
									self.assertTrue(isinstance(batch['input_ids']       ,np.ndarray     )     )
									self.assertEqual(batch['input_ids'].tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )







					@require_torch
					def 						__lowerCAmelCase		(							self  :      Any     ):
									import torch

									UpperCAmelCase__			 =				[{'input_ids': torch.tensor([0, 1, 2]     )}, {'input_ids': torch.tensor([0, 1, 2, 3]     )}]
									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )

									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__     )
									self.assertTrue(isinstance(batch['input_ids']       ,torch.Tensor     )     )
									self.assertEqual(batch['input_ids'].tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__       ,return_tensors='pt'     )
									self.assertTrue(isinstance(batch['input_ids']       ,torch.Tensor     )     )
									self.assertEqual(batch['input_ids'].tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )







					@require_tf
					def 						__lowerCAmelCase		(							self  :      Tuple     ):
									import tensorflow as tf

									UpperCAmelCase__			 =				[{'input_ids': tf.constant([0, 1, 2]     )}, {'input_ids': tf.constant([0, 1, 2, 3]     )}]
									UpperCAmelCase__			 =				BertTokenizer.from_pretrained('bert-base-cased'     )

									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__     )
									self.assertTrue(isinstance(batch['input_ids']       ,tf.Tensor     )     )
									self.assertEqual(batch['input_ids'].numpy().tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
									UpperCAmelCase__			 =				tokenizer.pad(lowerCamelCase__       ,padding=lowerCamelCase__       ,return_tensors='tf'     )
									self.assertTrue(isinstance(batch['input_ids']       ,tf.Tensor     )     )
									self.assertEqual(batch['input_ids'].numpy().tolist()       ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )







					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      Union[str, Any]     ):
									UpperCAmelCase__			 =				Tokenizer(WordPiece(unk_token='[UNK]'     )     )
									PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase__     )







					@require_tokenizers
					def 						__lowerCAmelCase		(							self  :      Any     ):
									UpperCAmelCase__			 =				Tokenizer(WordPiece(unk_token='[UNK]'     )     )
									with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(lowerCamelCase__       ,'tokenizer.json'     )     )
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase__       ,'tokenizer.json'     )     )




"""simple docstring"""






import numpy as np



def     a_		(   lowerCamelCase   ,     lowerCamelCase="utf-8"   ,     lowerCamelCase="surrogatepass"			):
				UpperCAmelCase__			 =				bin(int.from_bytes(text.encode(lowerCamelCase   ,     lowerCamelCase			)   ,     'big'			)			)[2:]
				return bits.zfill(8 * ((len(lowerCamelCase			) + 7) // 8)			)



def     a_		(   lowerCamelCase   ,     lowerCamelCase="utf-8"   ,     lowerCamelCase="surrogatepass"			):
				UpperCAmelCase__			 =				int(lowerCamelCase   ,     2			)
				return n.to_bytes((n.bit_length() + 7) // 8   ,     'big'			).decode(lowerCamelCase   ,     lowerCamelCase			) or "\0"



def     a_		(   lowerCamelCase   ,     lowerCamelCase			):
				if size_par + len(lowerCamelCase			) <= 2**size_par - (len(lowerCamelCase			) - 1):
								raise ValueError('size of parity don\'t match with size of data'			)

				UpperCAmelCase__			 =				[]
				UpperCAmelCase__			 =				[]
				UpperCAmelCase__			 =				[bin(lowerCamelCase			)[2:] for x in range(1   ,     size_par + len(lowerCamelCase			) + 1			)]

				# sorted information data for the size of the output data
				UpperCAmelCase__			 =				[]
				# data position template + parity
				UpperCAmelCase__			 =				[]
				# parity bit counter
				UpperCAmelCase__			 =				0
				# counter position of data bits
				UpperCAmelCase__			 =				0

				for x in range(1   ,     size_par + len(lowerCamelCase			) + 1			):
								# Performs a template of bit positions - who should be given,
								# and who should be parity
								if qtd_bp < size_par:
												if (np.log(lowerCamelCase			) / np.log(2			)).is_integer():
																data_out_gab.append('P'			)
																UpperCAmelCase__			 =				qtd_bp + 1
												else:
																data_out_gab.append('D'			)
								else:
												data_out_gab.append('D'			)

								# Sorts the data to the new output size
								if data_out_gab[-1] == "D":
												data_ord.append(data[cont_data]			)
												cont_data += 1
								else:
												data_ord.append(lowerCamelCase			)

    # Calculates parity
				UpperCAmelCase__			 =				0  # parity bit counter
				for bp in range(1   ,     size_par + 1			):
								# Bit counter one for a given parity
								UpperCAmelCase__			 =				0
								# counter to control the loop reading
								UpperCAmelCase__			 =				0
								for x in data_ord:
												if x is not None:
																try:
																				UpperCAmelCase__			 =				(bin_pos[cont_loop])[-1 * (bp)]
																except IndexError:
																				UpperCAmelCase__			 =				'0'
																if aux == "1" and x == "1":
																				cont_bo += 1
												cont_loop += 1
								parity.append(cont_bo % 2			)

								qtd_bp += 1

				# Mount the message
				UpperCAmelCase__			 =				0  # parity bit counter
				for x in range(0   ,     size_par + len(lowerCamelCase			)			):
								if data_ord[x] is None:
												data_out.append(str(parity[cont_bp]			)			)
												cont_bp += 1
								else:
												data_out.append(data_ord[x]			)

				return data_out



def     a_		(   lowerCamelCase   ,     lowerCamelCase			):
				UpperCAmelCase__			 =				[]
				# Parity bit counter
				UpperCAmelCase__			 =				0
				# Counter p data bit reading
				UpperCAmelCase__			 =				0
				# list of parity received
				UpperCAmelCase__			 =				[]
				UpperCAmelCase__			 =				[]

				for x in range(1   ,     len(lowerCamelCase			) + 1			):
								# Performs a template of bit positions - who should be given,
								#  and who should be parity
								if qtd_bp < size_par and (np.log(lowerCamelCase			) / np.log(2			)).is_integer():
												data_out_gab.append('P'			)
												UpperCAmelCase__			 =				qtd_bp + 1
								else:
												data_out_gab.append('D'			)

								# Sorts the data to the new output size
								if data_out_gab[-1] == "D":
												data_output.append(data[cont_data]			)
								else:
												parity_received.append(data[cont_data]			)
								cont_data += 1

				# -----------calculates the parity with the data
				UpperCAmelCase__			 =				[]
				UpperCAmelCase__			 =				[]
				UpperCAmelCase__			 =				[bin(lowerCamelCase			)[2:] for x in range(1   ,     size_par + len(lowerCamelCase			) + 1			)]

				#  sorted information data for the size of the output data
				UpperCAmelCase__			 =				[]
				# Data position feedback + parity
				UpperCAmelCase__			 =				[]
				# Parity bit counter
				UpperCAmelCase__			 =				0
				# Counter p data bit reading
				UpperCAmelCase__			 =				0

				for x in range(1   ,     size_par + len(lowerCamelCase			) + 1			):
								# Performs a template position of bits - who should be given,
								# and who should be parity
								if qtd_bp < size_par and (np.log(lowerCamelCase			) / np.log(2			)).is_integer():
												data_out_gab.append('P'			)
												UpperCAmelCase__			 =				qtd_bp + 1
								else:
												data_out_gab.append('D'			)

								# Sorts the data to the new output size
								if data_out_gab[-1] == "D":
												data_ord.append(data_output[cont_data]			)
												cont_data += 1
								else:
												data_ord.append(lowerCamelCase			)

    # Calculates parity
				UpperCAmelCase__			 =				0  # parity bit counter
				for bp in range(1   ,     size_par + 1			):
								# Bit counter one for a certain parity
								UpperCAmelCase__			 =				0
								# Counter to control loop reading
								UpperCAmelCase__			 =				0
								for x in data_ord:
												if x is not None:
																try:
																				UpperCAmelCase__			 =				(bin_pos[cont_loop])[-1 * (bp)]
																except IndexError:
																				UpperCAmelCase__			 =				'0'
																if aux == "1" and x == "1":
																				cont_bo += 1
												cont_loop += 1
								parity.append(str(cont_bo % 2			)			)

								qtd_bp += 1

				# Mount the message
				UpperCAmelCase__			 =				0  # Parity bit counter
				for x in range(0   ,     size_par + len(lowerCamelCase			)			):
								if data_ord[x] is None:
												data_out.append(str(parity[cont_bp]			)			)
												cont_bp += 1
								else:
												data_out.append(data_ord[x]			)

				UpperCAmelCase__			 =				parity_received == parity
				return data_output, ack



