


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

UpperCamelCase			:					Tuple         =							logging.get_logger(__name__)

UpperCamelCase			:					int         =							{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCamelCase			:					List[str]         =							{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

UpperCamelCase			:					int         =							{
    '''gpt2''': 1_024,
    '''gpt2-medium''': 1_024,
    '''gpt2-large''': 1_024,
    '''gpt2-xl''': 1_024,
    '''distilgpt2''': 1_024,
}
@lru_cache()
def 				UpperCamelCase_	(					)->   Optional[Any]:




						'''simple docstring'''
						__magic_name__					=      (
						    list(range(ord('!'  )						,      ord('~'  ) + 1  )  ) + list(range(ord('¡'  )						,      ord('¬'  ) + 1  )  ) + list(range(ord('®'  )						,      ord('ÿ'  ) + 1  )  )
						)
						__magic_name__					=      bs[:]
						__magic_name__					=      0
						for b in range(2**8  ):
												if b not in bs:
																		bs.append(__lowercase  )
																		cs.append(2**8 + n  )
																		n += 1
						__magic_name__					=      [chr(__lowercase  ) for n in cs]
						return dict(zip(__lowercase						,      __lowercase  )  )
def 				UpperCamelCase_	(					__snake_case      :    Optional[Any]  )->   List[Any]:




						'''simple docstring'''
						__magic_name__					=      set()
						__magic_name__					=      word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)  )
												__magic_name__					=      char
						return pairs




class      __snake_case				(   _a):
					SCREAMING_SNAKE_CASE_		:       str								=			VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE_		:       Dict								=			PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE_		:       int								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE_		:       Dict								=			["input_ids", "attention_mask"]
					def __init__(						self       :						Optional[Any]	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						Tuple="replace"	,      _lowerCAmelCase       :						Union[str, Any]="<|endoftext|>"	,      _lowerCAmelCase       :						int="<|endoftext|>"	,      _lowerCAmelCase       :						str="<|endoftext|>"	,      _lowerCAmelCase       :						Optional[Any]=None	,      _lowerCAmelCase       :						int=False	,      _lowerCAmelCase       :						Any=False	,      **_lowerCAmelCase       :						Optional[int]	,      )							->					Dict:



											'''simple docstring'''





											__magic_name__					=      AddedToken(_lowerCAmelCase	,      lstrip=_lowerCAmelCase	,      rstrip=_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else bos_token
											__magic_name__					=      AddedToken(_lowerCAmelCase	,      lstrip=_lowerCAmelCase	,      rstrip=_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else eos_token
											__magic_name__					=      AddedToken(_lowerCAmelCase	,      lstrip=_lowerCAmelCase	,      rstrip=_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else unk_token
											__magic_name__					=      AddedToken(_lowerCAmelCase	,      lstrip=_lowerCAmelCase	,      rstrip=_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else pad_token
											super().__init__(
											    errors=_lowerCAmelCase	,      unk_token=_lowerCAmelCase	,      bos_token=_lowerCAmelCase	,      eos_token=_lowerCAmelCase	,      pad_token=_lowerCAmelCase	,      add_prefix_space=_lowerCAmelCase	,      add_bos_token=_lowerCAmelCase	,      **_lowerCAmelCase	,      )
											__magic_name__					=      add_bos_token

											with open(_lowerCAmelCase	,      encoding='utf-8'  ) as vocab_handle:
																	__magic_name__					=      json.load(_lowerCAmelCase  )
											__magic_name__					=      {v: k for k, v in self.encoder.items()}
											__magic_name__					=      errors  # how to handle errors in decoding
											__magic_name__					=      bytes_to_unicode()
											__magic_name__					=      {v: k for k, v in self.byte_encoder.items()}
											with open(_lowerCAmelCase	,      encoding='utf-8'  ) as merges_handle:
																	__magic_name__					=      merges_handle.read().split('\n'  )[1:-1]
											__magic_name__					=      [tuple(merge.split()  ) for merge in bpe_merges]
											__magic_name__					=      dict(zip(_lowerCAmelCase	,      range(len(_lowerCAmelCase  )  )  )  )
											__magic_name__					=      {}
											__magic_name__					=      add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											__magic_name__					=      re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'  )
					@property
					def 	a			(						self       :						int  )							->					Any:



											'''simple docstring'''





											return len(self.encoder  )
					def 	a			(						self       :						Dict  )							->					Optional[int]:



											'''simple docstring'''





											return dict(self.encoder	,      **self.added_tokens_encoder  )
					def 	a			(						self       :						int	,      _lowerCAmelCase       :						str  )							->					Optional[Any]:



											'''simple docstring'''





											if token in self.cache:
																	return self.cache[token]
											__magic_name__					=      tuple(_lowerCAmelCase  )
											__magic_name__					=      get_pairs(_lowerCAmelCase  )

											if not pairs:
																	return token

											while True:
																	__magic_name__					=      min(_lowerCAmelCase	,      key=lambda _lowerCAmelCase  : self.bpe_ranks.get(_lowerCAmelCase	,      float('inf'  )  )  )
																	if bigram not in self.bpe_ranks:
																							break
																	__magic_name__		,      __magic_name__					=      bigram
																	__magic_name__					=      []
																	__magic_name__					=      0
																	while i < len(_lowerCAmelCase  ):
																							try:
																													__magic_name__					=      word.index(_lowerCAmelCase	,      _lowerCAmelCase  )
																							except ValueError:
																													new_word.extend(word[i:]  )
																													break
																							else:
																													new_word.extend(word[i:j]  )
																													__magic_name__					=      j

																							if word[i] == first and i < len(_lowerCAmelCase  ) - 1 and word[i + 1] == second:
																													new_word.append(first + second  )
																													i += 2
																							else:
																													new_word.append(word[i]  )
																													i += 1
																	__magic_name__					=      tuple(_lowerCAmelCase  )
																	__magic_name__					=      new_word
																	if len(_lowerCAmelCase  ) == 1:
																							break
																	else:
																							__magic_name__					=      get_pairs(_lowerCAmelCase  )
											__magic_name__					=      ' '.join(_lowerCAmelCase  )
											__magic_name__					=      word
											return word
					def 	a			(						self       :						Optional[int]	,      _lowerCAmelCase       :						Dict	,      _lowerCAmelCase       :						Optional[Any]=None  )							->					Tuple:



											'''simple docstring'''





											if self.add_bos_token:
																	__magic_name__					=      [self.bos_token_id]
											else:
																	__magic_name__					=      []

											__magic_name__					=      bos_token_ids + token_ids_a

											if token_ids_a is None:
																	return output

											return output + bos_token_ids + token_ids_a
					def 	a			(						self       :						Optional[int]	,      _lowerCAmelCase       :						List[int]	,      _lowerCAmelCase       :						Optional[List[int]] = None	,      _lowerCAmelCase       :						bool = False  )							->					str:



											'''simple docstring'''





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_lowerCAmelCase	,      token_ids_a=_lowerCAmelCase	,      already_has_special_tokens=_lowerCAmelCase  )

											if not self.add_bos_token:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_lowerCAmelCase	,      token_ids_a=_lowerCAmelCase	,      already_has_special_tokens=_lowerCAmelCase  )

											if token_ids_a is None:
																	return [1] + ([0] * len(_lowerCAmelCase  ))
											return [1] + ([0] * len(_lowerCAmelCase  )) + [1] + ([0] * len(_lowerCAmelCase  ))
					def 	a			(						self       :						int	,      _lowerCAmelCase       :						Tuple  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      []
											for token in re.findall(self.pat	,      _lowerCAmelCase  ):
																	__magic_name__					=      ''.join(
																	    self.byte_encoder[b] for b in token.encode('utf-8'  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCAmelCase  ).split(' '  )  )
											return bpe_tokens
					def 	a			(						self       :						Any	,      _lowerCAmelCase       :						Any  )							->					int:



											'''simple docstring'''





											return self.encoder.get(_lowerCAmelCase	,      self.encoder.get(self.unk_token  )  )
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						Optional[Any]  )							->					str:



											'''simple docstring'''





											return self.decoder.get(_lowerCAmelCase  )
					def 	a			(						self       :						Union[str, Any]	,      _lowerCAmelCase       :						List[str]  )							->					List[Any]:



											'''simple docstring'''





											__magic_name__					=      ''.join(_lowerCAmelCase  )
											__magic_name__					=      bytearray([self.byte_decoder[c] for c in text]  ).decode('utf-8'	,      errors=self.errors  )
											return text
					def 	a			(						self       :						Optional[Any]	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						Optional[str] = None  )							->					Union[str, Any]:



											'''simple docstring'''





											if not os.path.isdir(_lowerCAmelCase  ):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
																	return
											__magic_name__					=      os.path.join(
											    _lowerCAmelCase	,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
											__magic_name__					=      os.path.join(
											    _lowerCAmelCase	,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

											with open(_lowerCAmelCase	,      'w'	,      encoding='utf-8'  ) as f:
																	f.write(json.dumps(self.encoder	,      indent=2	,      sort_keys=_lowerCAmelCase	,      ensure_ascii=_lowerCAmelCase  ) + '\n'  )

											__magic_name__					=      0
											with open(_lowerCAmelCase	,      'w'	,      encoding='utf-8'  ) as writer:
																	writer.write('#version: 0.2\n'  )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,      key=lambda _lowerCAmelCase  : kv[1]  ):
																							if index != token_index:
																													logger.warning(
																													    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    ' Please check that the tokenizer is not corrupted!'  )
																													__magic_name__					=      token_index
																							writer.write(' '.join(_lowerCAmelCase  ) + '\n'  )
																							index += 1

											return vocab_file, merge_file
					def 	a			(						self       :						Any	,      _lowerCAmelCase       :						Optional[Any]	,      _lowerCAmelCase       :						Optional[int]=False	,      **_lowerCAmelCase       :						Optional[int]  )							->					List[Any]:



											'''simple docstring'''





											__magic_name__					=      kwargs.pop('add_prefix_space'	,      self.add_prefix_space  )
											if is_split_into_words or add_prefix_space:
																	__magic_name__					=      ' ' + text
											return (text, kwargs)







					def 	a			(						self       :						Optional[Any]	,      _lowerCAmelCase       :						"Conversation"  )							->					Optional[int]:



											'''simple docstring'''





											__magic_name__					=      []
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  ) + [self.eos_token_id]  )
											if len(_lowerCAmelCase  ) > self.model_max_length:
																	__magic_name__					=      input_ids[-self.model_max_length :]
											return input_ids








import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


__lowercase				    =      (30, 487)
__lowercase				    =      (36, 1_024)
__lowercase				    =      (100, 100)
__lowercase				    =      100

__lowercase				    =      datasets.Features(
    {"""text""": ArrayaD(SHAPE_TEST_1, dtype="""float32"""), """image""": ArrayaD(SHAPE_TEST_2, dtype="""float32""")}
)

__lowercase    ,							__lowercase				    =      os.path.split(__file__)
__lowercase				    =      os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		Any  ,				__lowercase:		Optional[Any]  ,				__lowercase:		List[Any]):



  '''simple docstring'''






  with ArrowWriter(features=__lowercase  ,				path=os.path.join(__lowercase  ,				'''beta.arrow''')) as writer:
    for key, record in dummy_data:
      __lowerCAmelCase								=					my_features.encode_example(__lowercase)
      writer.write(__lowercase)
    __lowerCAmelCase						,    __lowerCAmelCase								=					writer.finalize()


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		str  ,				__lowercase:		str):



  '''simple docstring'''






  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  for _ in dataset:
    pass


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		Tuple  ,				__lowercase:		str):



  '''simple docstring'''






  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  dataset.set_format('''numpy''')
  for _ in dataset:
    pass


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		str  ,				__lowercase:		Dict):



  '''simple docstring'''






  __lowerCAmelCase								=					10
  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  for i in range(0  ,				len(__lowercase)  ,				__lowercase):
    __lowerCAmelCase								=					dataset[i : i + batch_size]


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		List[Any]  ,				__lowercase:		Union[str, Any]):



  '''simple docstring'''






  __lowerCAmelCase								=					10
  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  dataset.set_format('''numpy''')
  for i in range(0  ,				len(__lowercase)  ,				__lowercase):
    __lowerCAmelCase								=					dataset[i : i + batch_size]


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		Union[str, Any]  ,				__lowercase:		str):



  '''simple docstring'''






  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  for col in feats:
    __lowerCAmelCase								=					dataset[col]


@get_duration
def        __SCREAMING_SNAKE_CASE	(				__lowercase:		Optional[int]  ,				__lowercase:		Optional[Any]):



  '''simple docstring'''






  __lowerCAmelCase								=					datasets.Dataset.from_file(
      filename=os.path.join(__lowercase  ,				'''beta.arrow''')  ,				info=datasets.DatasetInfo(features=__lowercase))
  dataset.set_format('''numpy''')
  for col in feats:
    __lowerCAmelCase								=					dataset[col]


def        __SCREAMING_SNAKE_CASE	(				):



  '''simple docstring'''






  __lowerCAmelCase								=					{}
  __lowerCAmelCase								=					(
      read_unformated,
      read_formatted_as_numpy,
      read_batch_unformated,
      read_batch_formatted_as_numpy,
      read_col_unformated,
      read_col_formatted_as_numpy,
  )
  with tempfile.TemporaryDirectory() as tmp_dir:
    __lowerCAmelCase								=					datasets.Features({'''image''': ArrayaD(__lowercase  ,				dtype='''float32''')})
    __lowerCAmelCase								=					generate_examples(features=__lowercase  ,				num_examples=__lowercase)
    __lowerCAmelCase								=					write(__lowercase  ,				__lowercase  ,				__lowercase)
    for read_func in read_functions:
      __lowerCAmelCase								=					read_func(__lowercase  ,				__lowercase)

  with tempfile.TemporaryDirectory() as tmp_dir:
    # don't use fixed length for fair comparison
    # feats = datasets.Features(
    #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
    # )
    __lowerCAmelCase								=					datasets.Features({'''image''': datasets.Sequence(datasets.Sequence(datasets.Value('''float32''')))})
    __lowerCAmelCase								=					generate_examples(
        features=__lowercase  ,				num_examples=__lowercase  ,				seq_shapes={'''image''': SPEED_TEST_SHAPE})
    __lowerCAmelCase								=					write(__lowercase  ,				__lowercase  ,				__lowercase)
    for read_func in read_functions:
      __lowerCAmelCase								=					read_func(__lowercase  ,				__lowercase)

  with tempfile.TemporaryDirectory() as tmp_dir:
    # don't use fixed length for fair comparison
    # feats = datasets.Features(
    #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
    # )
    __lowerCAmelCase								=					datasets.Features({'''image''': datasets.Sequence(datasets.Value('''float32'''))})
    __lowerCAmelCase								=					generate_examples(
        features=__lowercase  ,				num_examples=__lowercase  ,				seq_shapes={'''image''': [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}  ,				)
    __lowerCAmelCase								=					write(__lowercase  ,				__lowercase  ,				__lowercase)
    for read_func in read_functions:
      __lowerCAmelCase								=					read_func(__lowercase  ,				__lowercase)

  with open(__lowercase  ,				'''wb''') as f:
    f.write(json.dumps(__lowercase).encode('''utf-8'''))


if __name__ == "__main__":  # useful to run the profiler
      benchmark_array_xd()
