




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_   							=      logging.get_logger(__name__)

UpperCAmelCase_   							=      {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase_   							=      {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

UpperCAmelCase_   							=      {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}




@lru_cache()
def    lowerCamelCase__			(		):





       '''simple docstring'''



       __lowerCamelCase  =		(
           list(range(ord("""!"""      )    ,     ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )    ,     ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )    ,     ord("""ÿ"""      ) + 1      )      )
       )
       __lowerCamelCase  =		bs[:]
       __lowerCamelCase  =		0
       for b in range(2**8      ):
              if b not in bs:
                     bs.append(A__      )
                     cs.append(2**8 + n      )
                     n += 1
       __lowerCamelCase  =		[chr(A__      ) for n in cs]
       return dict(zip(A__    ,     A__      )      )




def    lowerCamelCase__			(		A__			:   Any      ):





       '''simple docstring'''



       __lowerCamelCase  =		set()
       __lowerCamelCase  =		word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)      )
              __lowerCamelCase  =		char
       return pairs





class        lowerCamelCase__(   __lowerCamelCase):
      UpperCAmelCase__ :      Tuple       						=  VOCAB_FILES_NAMES
      UpperCAmelCase__ :      Any       						=  PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__ :      Tuple       						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__ :      str       						=  ['input_ids', 'attention_mask']
      def __init__(							self:					Dict       ,			UpperCamelCase_:					List[Any]       ,			UpperCamelCase_:					Tuple       ,			UpperCamelCase_:					Tuple="replace"       ,			UpperCamelCase_:					List[Any]="<s>"       ,			UpperCamelCase_:					Dict="</s>"       ,			UpperCamelCase_:					Union[str, Any]="</s>"       ,			UpperCamelCase_:					Optional[int]="<s>"       ,			UpperCamelCase_:					str="<unk>"       ,			UpperCamelCase_:					str="<pad>"       ,			UpperCamelCase_:					Optional[int]="<mask>"       ,			UpperCamelCase_:					Tuple=False       ,			**UpperCamelCase_:					Tuple       ,			):
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else bos_token
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else eos_token
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else sep_token
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else cls_token
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else unk_token
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             __lowerCamelCase  =		AddedToken(UpperCamelCase_       ,			lstrip=UpperCamelCase_       ,			rstrip=UpperCamelCase_  ) if isinstance(UpperCamelCase_       ,			UpperCamelCase_  ) else mask_token

             super().__init__(
                 errors=UpperCamelCase_       ,			bos_token=UpperCamelCase_       ,			eos_token=UpperCamelCase_       ,			unk_token=UpperCamelCase_       ,			sep_token=UpperCamelCase_       ,			cls_token=UpperCamelCase_       ,			pad_token=UpperCamelCase_       ,			mask_token=UpperCamelCase_       ,			add_prefix_space=UpperCamelCase_       ,			**UpperCamelCase_       ,			)

             with open(UpperCamelCase_       ,			encoding="""utf-8"""  ) as vocab_handle:
                    __lowerCamelCase  =		json.load(UpperCamelCase_  )
             __lowerCamelCase  =		{v: k for k, v in self.encoder.items()}
             __lowerCamelCase  =		errors  # how to handle errors in decoding
             __lowerCamelCase  =		bytes_to_unicode()
             __lowerCamelCase  =		{v: k for k, v in self.byte_encoder.items()}
             with open(UpperCamelCase_       ,			encoding="""utf-8"""  ) as merges_handle:
                    __lowerCamelCase  =		merges_handle.read().split("""\n"""  )[1:-1]
             __lowerCamelCase  =		[tuple(merge.split()  ) for merge in bpe_merges]
             __lowerCamelCase  =		dict(zip(UpperCamelCase_       ,			range(len(UpperCamelCase_  )  )  )  )
             __lowerCamelCase  =		{}
             __lowerCamelCase  =		add_prefix_space

             # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
             __lowerCamelCase  =		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )
      @property
      def      lowerCAmelCase__		(							self:					Dict  ):
             return len(self.encoder  )
      def      lowerCAmelCase__		(							self:					Optional[Any]  ):
             return dict(self.encoder       ,			**self.added_tokens_encoder  )
      def      lowerCAmelCase__		(							self:					List[Any]       ,			UpperCamelCase_:					List[str]  ):
             if token in self.cache:
                    return self.cache[token]
             __lowerCamelCase  =		tuple(UpperCamelCase_  )
             __lowerCamelCase  =		get_pairs(UpperCamelCase_  )

             if not pairs:
                    return token

             while True:
                    __lowerCamelCase  =		min(UpperCamelCase_       ,			key=lambda UpperCamelCase_  : self.bpe_ranks.get(UpperCamelCase_       ,			float("""inf"""  )  )  )
                    if bigram not in self.bpe_ranks:
                           break
                    __lowerCamelCase,  __lowerCamelCase  =		bigram
                    __lowerCamelCase  =		[]
                    __lowerCamelCase  =		0
                    while i < len(UpperCamelCase_  ):
                           try:
                                  __lowerCamelCase  =		word.index(UpperCamelCase_       ,			UpperCamelCase_  )
                           except ValueError:
                                  new_word.extend(word[i:]  )
                                  break
                           else:
                                  new_word.extend(word[i:j]  )
                                  __lowerCamelCase  =		j

                           if word[i] == first and i < len(UpperCamelCase_  ) - 1 and word[i + 1] == second:
                                  new_word.append(first + second  )
                                  i += 2
                           else:
                                  new_word.append(word[i]  )
                                  i += 1
                    __lowerCamelCase  =		tuple(UpperCamelCase_  )
                    __lowerCamelCase  =		new_word
                    if len(UpperCamelCase_  ) == 1:
                           break
                    else:
                           __lowerCamelCase  =		get_pairs(UpperCamelCase_  )
             __lowerCamelCase  =		""" """.join(UpperCamelCase_  )
             __lowerCamelCase  =		word
             return word
      def      lowerCAmelCase__		(							self:					Dict       ,			UpperCamelCase_:					int  ):
             __lowerCamelCase  =		[]
             for token in re.findall(self.pat       ,			UpperCamelCase_  ):
                    __lowerCamelCase  =		"""""".join(
                        self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                    bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_  ).split(""" """  )  )
             return bpe_tokens
      def      lowerCAmelCase__		(							self:					Union[str, Any]       ,			UpperCamelCase_:					List[str]  ):
             return self.encoder.get(UpperCamelCase_       ,			self.encoder.get(self.unk_token  )  )
      def      lowerCAmelCase__		(							self:					Optional[Any]       ,			UpperCamelCase_:					Optional[int]  ):
             return self.decoder.get(UpperCamelCase_  )
      def      lowerCAmelCase__		(							self:					Optional[Any]       ,			UpperCamelCase_:					List[Any]  ):
             __lowerCamelCase  =		"""""".join(UpperCamelCase_  )
             __lowerCamelCase  =		bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8"""       ,			errors=self.errors  )
             return text
      def      lowerCAmelCase__		(							self:					str       ,			UpperCamelCase_:					str       ,			UpperCamelCase_:					Optional[str] = None  ):
             if not os.path.isdir(UpperCamelCase_  ):
                    logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
                    return
             __lowerCamelCase  =		os.path.join(
                 UpperCamelCase_       ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
             __lowerCamelCase  =		os.path.join(
                 UpperCamelCase_       ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

             with open(UpperCamelCase_       ,			"""w"""       ,			encoding="""utf-8"""  ) as f:
                    f.write(json.dumps(self.encoder       ,			indent=2       ,			sort_keys=UpperCamelCase_       ,			ensure_ascii=UpperCamelCase_  ) + """\n"""  )

             __lowerCamelCase  =		0
             with open(UpperCamelCase_       ,			"""w"""       ,			encoding="""utf-8"""  ) as writer:
                    writer.write("""#version: 0.2\n"""  )
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,			key=lambda UpperCamelCase_  : kv[1]  ):
                           if index != token_index:
                                  logger.warning(
                                      F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                      """ Please check that the tokenizer is not corrupted!"""  )
                                  __lowerCamelCase  =		token_index
                           writer.write(""" """.join(UpperCamelCase_  ) + """\n"""  )
                           index += 1

             return vocab_file, merge_file
      def      lowerCAmelCase__		(							self:					Tuple       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None  ):
             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             __lowerCamelCase  =		[self.cls_token_id]
             __lowerCamelCase  =		[self.sep_token_id]
             return cls + token_ids_a + sep + sep + token_ids_a + sep
      def      lowerCAmelCase__		(							self:					Dict       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None       ,			UpperCamelCase_:					bool = False  ):
             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase_       ,			token_ids_a=UpperCamelCase_       ,			already_has_special_tokens=UpperCamelCase_  )

             if token_ids_a is None:
                    return [1] + ([0] * len(UpperCamelCase_  )) + [1]
             return [1] + ([0] * len(UpperCamelCase_  )) + [1, 1] + ([0] * len(UpperCamelCase_  )) + [1]
      def      lowerCAmelCase__		(							self:					List[str]       ,			UpperCamelCase_:					List[int]       ,			UpperCamelCase_:					Optional[List[int]] = None  ):
             __lowerCamelCase  =		[self.sep_token_id]
             __lowerCamelCase  =		[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep  ) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]




      def      lowerCAmelCase__		(							self:					Tuple       ,			UpperCamelCase_:					Tuple       ,			UpperCamelCase_:					Optional[Any]=False       ,			**UpperCamelCase_:					Any  ):
             __lowerCamelCase  =		kwargs.pop("""add_prefix_space"""       ,			self.add_prefix_space  )
             if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_  ) > 0 and not text[0].isspace()):
                    __lowerCamelCase  =		""" """ + text
             return (text, kwargs)







from __future__ import annotations

from collections.abc import Iterable





class        lowerCamelCase__:
      def __init__(							self:					Dict       ,			UpperCamelCase_:					int       ,			UpperCamelCase_:					str  ):
             __lowerCamelCase,  __lowerCamelCase  =		float(UpperCamelCase_  ), float(UpperCamelCase_  )
      def __eq__(							self:					List[Any]       ,			UpperCamelCase_:					List[str]  ):
             return self.x == other.x and self.y == other.y
      def __ne__(							self:					Tuple       ,			UpperCamelCase_:					Dict  ):
             return not self == other
      def __gt__(							self:					List[Any]       ,			UpperCamelCase_:					List[Any]  ):
             if self.x > other.x:
                    return True
             elif self.x == other.x:
                    return self.y > other.y
             return False
      def __lt__(							self:					str       ,			UpperCamelCase_:					int  ):
             return not self > other
      def __ge__(							self:					Dict       ,			UpperCamelCase_:					Any  ):
             if self.x > other.x:
                    return True
             elif self.x == other.x:
                    return self.y >= other.y
             return False
      def __le__(							self:					Union[str, Any]       ,			UpperCamelCase_:					Optional[Any]  ):
             if self.x < other.x:
                    return True
             elif self.x == other.x:
                    return self.y <= other.y
             return False
      def __repr__(							self:					Union[str, Any]  ):
             return F'({self.x}, {self.y})'




      def __hash__(							self:					List[Any]  ):
             return hash(self.x  )




def    lowerCamelCase__			(		A__			:   list[Point] | list[list[float]] | Iterable[list[float]]    ,     ):





       '''simple docstring'''



       __lowerCamelCase  =		[]
       if list_of_tuples:
              for p in list_of_tuples:
                     if isinstance(A__    ,     A__      ):
                            points.append(A__      )
                     else:
                            try:
                                   points.append(Point(p[0]    ,     p[1]      )      )
                            except (IndexError, TypeError):
                                   print(
                                       f'Ignoring deformed point {p}. All points'
                                       """ must have at least 2 coordinates."""      )
       return points




def    lowerCamelCase__			(		A__			:   list[Point] | list[list[float]]      ):





       '''simple docstring'''



       if not hasattr(A__    ,     """__iter__"""      ):
              __lowerCamelCase  =		f'Expecting an iterable object but got an non-iterable type {points}'
              raise ValueError(A__      )

       if not points:
              __lowerCamelCase  =		f'Expecting a list of points but got {points}'
              raise ValueError(A__      )

       return _construct_points(A__      )




def    lowerCamelCase__			(		A__			:   Point    ,     A__			:   Point    ,     A__			:   Point      ):





       '''simple docstring'''



       __lowerCamelCase  =		(a.x * b.y + b.x * c.y + c.x * a.y) - (a.y * b.x + b.y * c.x + c.y * a.x)
       return det




def    lowerCamelCase__			(		A__			:   list[Point]      ):





       '''simple docstring'''



       __lowerCamelCase  =		sorted(_validate_input(A__      )      )
       __lowerCamelCase  =		len(A__      )
       __lowerCamelCase  =		set()

       for i in range(n - 1      ):
              for j in range(i + 1    ,     A__      ):
                     __lowerCamelCase  =		__lowerCamelCase  =		False
                     __lowerCamelCase  =		True
                     for k in range(A__      ):
                            if k != i and k != j:
                                   __lowerCamelCase  =		_det(points[i]    ,     points[j]    ,     points[k]      )

                                   if det_k > 0:
                                          __lowerCamelCase  =		True
                                   elif det_k < 0:
                                          __lowerCamelCase  =		True
                                   else:
                                          # point[i], point[j], point[k] all lie on a straight line
                                          # if point[k] is to the left of point[i] or it's to the
                                          # right of point[j], then point[i], point[j] cannot be
                                          # part of the convex hull of A
                                          if points[k] < points[i] or points[k] > points[j]:
                                                 __lowerCamelCase  =		False
                                                 break

                            if points_left_of_ij and points_right_of_ij:
                                   __lowerCamelCase  =		False
                                   break

                     if ij_part_of_convex_hull:
                            convex_set.update([points[i], points[j]]      )

       return sorted(A__      )




def    lowerCamelCase__			(		A__			:   list[Point]      ):





       '''simple docstring'''



       __lowerCamelCase  =		sorted(_validate_input(A__      )      )
       __lowerCamelCase  =		len(A__      )

       # divide all the points into an upper hull and a lower hull
       # the left most point and the right most point are definitely
       # members of the convex hull by definition.
       # use these two anchors to divide all the points into two hulls,
       # an upper hull and a lower hull.

       # all points to the left (above) the line joining the extreme points belong to the
       # upper hull
       # all points to the right (below) the line joining the extreme points below to the
       # lower hull
       # ignore all points on the line joining the extreme points since they cannot be
       # part of the convex hull

       __lowerCamelCase  =		points[0]
       __lowerCamelCase  =		points[n - 1]

       __lowerCamelCase  =		{left_most_point, right_most_point}
       __lowerCamelCase  =		[]
       __lowerCamelCase  =		[]

       for i in range(1    ,     n - 1      ):
              __lowerCamelCase  =		_det(A__    ,     A__    ,     points[i]      )

              if det > 0:
                     upper_hull.append(points[i]      )
              elif det < 0:
                     lower_hull.append(points[i]      )

       _construct_hull(A__    ,     A__    ,     A__    ,     A__      )
       _construct_hull(A__    ,     A__    ,     A__    ,     A__      )

       return sorted(A__      )




def    lowerCamelCase__			(		A__			:   list[Point]    ,     A__			:   Point    ,     A__			:   Point    ,     A__			:   set[Point]      ):





       '''simple docstring'''



       if points:
              __lowerCamelCase  =		None
              __lowerCamelCase  =		float("""-inf"""      )
              __lowerCamelCase  =		[]

              for p in points:
                     __lowerCamelCase  =		_det(A__    ,     A__    ,     A__      )

                     if det > 0:
                            candidate_points.append(A__      )

                            if det > extreme_point_distance:
                                   __lowerCamelCase  =		det
                                   __lowerCamelCase  =		p

              if extreme_point:
                     _construct_hull(A__    ,     A__    ,     A__    ,     A__      )
                     convex_set.add(A__      )
                     _construct_hull(A__    ,     A__    ,     A__    ,     A__      )




def    lowerCamelCase__			(		A__			:   list[Point]      ):





       '''simple docstring'''



       __lowerCamelCase  =		sorted(_validate_input(A__      )      )
       __lowerCamelCase  =		len(A__      )

       __lowerCamelCase  =		points[:2]
       for i in range(2    ,     A__      ):
              __lowerCamelCase  =		_det(convex_hull[1]    ,     convex_hull[0]    ,     points[i]      )
              if det > 0:
                     convex_hull.insert(0    ,     points[i]      )
                     break
              elif det < 0:
                     convex_hull.append(points[i]      )
                     break
              else:
                     __lowerCamelCase  =		points[i]
       i += 1

       for j in range(A__    ,     A__      ):
              if (
                  _det(convex_hull[0]    ,     convex_hull[-1]    ,     points[j]      ) > 0
                  and _det(convex_hull[-1]    ,     convex_hull[0]    ,     points[1]      ) < 0
              ):
                     # The point lies within the convex hull
                     continue

              convex_hull.insert(0    ,     points[j]      )
              convex_hull.append(points[j]      )
              while _det(convex_hull[0]    ,     convex_hull[1]    ,     convex_hull[2]      ) >= 0:
                     del convex_hull[1]
              while _det(convex_hull[-1]    ,     convex_hull[-2]    ,     convex_hull[-3]      ) <= 0:
                     del convex_hull[-2]

    # `convex_hull` is contains the convex hull in circular order
       return sorted(convex_hull[1:] if len(A__      ) > 3 else convex_hull      )




def    lowerCamelCase__			(		):





       '''simple docstring'''



       __lowerCamelCase  =		[
           (0, 3),
           (2, 2),
           (1, 1),
           (2, 1),
           (3, 0),
           (0, 0),
           (3, 3),
           (2, -1),
           (2, -4),
           (1, -3),
       ]
       # the convex set of points is
       # [(0, 0), (0, 3), (1, -3), (2, -4), (3, 0), (3, 3)]
       __lowerCamelCase  =		convex_hull_bf(A__      )

       __lowerCamelCase  =		convex_hull_recursive(A__      )
       assert results_bf == results_recursive

       __lowerCamelCase  =		convex_hull_melkman(A__      )
       assert results_bf == results_melkman

       print(A__      )


if __name__ == "__main__":
       main()

