

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a		       =   logging.get_logger(__name__)

_a		       =   {"vocab_file": "sentencepiece.bpe.model"}

_a		       =   {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    }
}

_a		       =   {
    "camembert-base": 5_1_2,
}

_a		       =   "▁"






class 							A_ (		_a					):
     _lowercase			:     List[str]								=	VOCAB_FILES_NAMES
     _lowercase			:     Any								=	PRETRAINED_VOCAB_FILES_MAP
     _lowercase			:     List[str]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowercase			:     List[Any]								=	['input_ids', 'attention_mask']


     def __init__(							self			:		Any			,  UpperCAmelCase			:		List[Any]			,  UpperCAmelCase			:		Tuple="<s>"			,  UpperCAmelCase			:		int="</s>"			,  UpperCAmelCase			:		str="</s>"			,  UpperCAmelCase			:		Optional[Any]="<s>"			,  UpperCAmelCase			:		Any="<unk>"			,  UpperCAmelCase			:		Optional[int]="<pad>"			,  UpperCAmelCase			:		Optional[Any]="<mask>"			,  UpperCAmelCase			:		Tuple=["<s>NOTUSED", "</s>NOTUSED"]			,  UpperCAmelCase			:		Optional[Dict[str, Any]] = None			,  **UpperCAmelCase			:		Optional[Any]			,  )							->	str:
      __lowerCAmelCase:   Optional[int]   	=      AddedToken(UpperCamelCase__			,  lstrip=UpperCamelCase__			,  rstrip=UpperCamelCase__					) if isinstance(UpperCamelCase__			,  UpperCamelCase__					) else mask_token

      __lowerCAmelCase:   Dict   	=      {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=UpperCamelCase__			,  eos_token=UpperCamelCase__			,  unk_token=UpperCamelCase__			,  sep_token=UpperCamelCase__			,  cls_token=UpperCamelCase__			,  pad_token=UpperCamelCase__			,  mask_token=UpperCamelCase__			,  additional_special_tokens=UpperCamelCase__			,  sp_model_kwargs=self.sp_model_kwargs			,  **UpperCamelCase__			,  )
      __lowerCAmelCase:   int   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
      self.sp_model.Load(str(UpperCamelCase__					)					)
      __lowerCAmelCase:   Dict   	=      vocab_file
      # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
      # sentencepiece vocabulary (this is the case for <s> and </s>
      __lowerCAmelCase:   Tuple   	=      {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
      __lowerCAmelCase:   Any   	=      len(self.fairseq_tokens_to_ids					)
      __lowerCAmelCase:   List[str]   	=      len(self.sp_model					) + len(self.fairseq_tokens_to_ids					)
      __lowerCAmelCase:   Optional[int]   	=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}


     def   UpperCAmelCase  (							self			:		List[Any]			,  UpperCAmelCase			:		List[int]			,  UpperCAmelCase			:		Optional[List[int]] = None					)							->	str:







      if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      __lowerCAmelCase:   Optional[Any]   	=      [self.cls_token_id]
      __lowerCAmelCase:   Tuple   	=      [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep


     def   UpperCAmelCase  (							self			:		List[Any]			,  UpperCAmelCase			:		List[int]			,  UpperCAmelCase			:		Optional[List[int]] = None			,  UpperCAmelCase			:		bool = False					)							->	Optional[Any]:







      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=UpperCamelCase__			,  token_ids_a=UpperCamelCase__			,  already_has_special_tokens=UpperCamelCase__					)

      if token_ids_a is None:
       return [1] + ([0] * len(UpperCamelCase__					)) + [1]
      return [1] + ([0] * len(UpperCamelCase__					)) + [1, 1] + ([0] * len(UpperCamelCase__					)) + [1]


     def   UpperCAmelCase  (							self			:		Optional[int]			,  UpperCAmelCase			:		List[int]			,  UpperCAmelCase			:		Optional[List[int]] = None					)							->	List[str]:
      __lowerCAmelCase:   Tuple   	=      [self.sep_token_id]
      __lowerCAmelCase:   Union[str, Any]   	=      [self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep					) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]


     @property
     def   UpperCAmelCase  (							self			:		str					)							->	Any:
      return len(self.fairseq_tokens_to_ids					) + len(self.sp_model					)


     def   UpperCAmelCase  (							self			:		Any					)							->	Optional[int]:
      __lowerCAmelCase:   Any   	=      {self.convert_ids_to_tokens(UpperCamelCase__					): i for i in range(self.vocab_size					)}
      vocab.update(self.added_tokens_encoder					)
      return vocab


     def   UpperCAmelCase  (							self			:		int			,  UpperCAmelCase			:		str					)							->	str:
      return self.sp_model.encode(UpperCamelCase__			,  out_type=UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		str			,  UpperCAmelCase			:		List[str]					)							->	Optional[int]:







      if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
      elif self.sp_model.PieceToId(UpperCamelCase__					) == 0:
       # Convert sentence piece unk token to fairseq unk token index
       return self.unk_token_id
      return self.fairseq_offset + self.sp_model.PieceToId(UpperCamelCase__					)


     def   UpperCAmelCase  (							self			:		str			,  UpperCAmelCase			:		Any					)							->	Union[str, Any]:







      if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset					)


     def   UpperCAmelCase  (							self			:		Union[str, Any]			,  UpperCAmelCase			:		List[Any]					)							->	Any:
      __lowerCAmelCase:   Optional[int]   	=      []
      __lowerCAmelCase:   Any   	=      ''
      __lowerCAmelCase:   Optional[Any]   	=      False
      for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
        if not prev_is_special:
         out_string += " "
        out_string += self.sp_model.decode(UpperCamelCase__					) + token
        __lowerCAmelCase:   Tuple   	=      True
        __lowerCAmelCase:   int   	=      []
       else:
        current_sub_tokens.append(UpperCamelCase__					)
        __lowerCAmelCase:   Optional[int]   	=      False
      out_string += self.sp_model.decode(UpperCamelCase__					)
      return out_string.strip()


     def __getstate__(							self			:		List[Any]					)							->	str:
      __lowerCAmelCase:   Optional[Any]   	=      self.__dict__.copy()
      __lowerCAmelCase:   Union[str, Any]   	=      None
      return state


     def __setstate__(							self			:		Union[str, Any]			,  UpperCAmelCase			:		Any					)							->	Any:
      __lowerCAmelCase:   Dict   	=      d

      # for backward compatibility
      if not hasattr(self			,  'sp_model_kwargs'					):
       __lowerCAmelCase:   str   	=      {}

      __lowerCAmelCase:   Tuple   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
      self.sp_model.Load(self.vocab_file					)


     def   UpperCAmelCase  (							self			:		Optional[Any]			,  UpperCAmelCase			:		str			,  UpperCAmelCase			:		Optional[str] = None					)							->	Tuple:







      if not os.path.isdir(UpperCamelCase__					):
       logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
       return
      __lowerCAmelCase:   str   	=      os.path.join(
          UpperCamelCase__			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

      if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase__					) and os.path.isfile(self.vocab_file					):
       copyfile(self.vocab_file			,  UpperCamelCase__					)
      elif not os.path.isfile(self.vocab_file					):
       with open(UpperCamelCase__			,  'wb'					) as fi:
        __lowerCAmelCase:   List[Any]   	=      self.sp_model.serialized_model_proto()
        fi.write(UpperCamelCase__					)

      return (out_vocab_file,)










'''simple docstring'''


import os
from itertools import chain
from random import randrange, shuffle

import pytest

from .sola import PokerHand

_lowerCamelCase	:       Optional[int]      	=	(
    "4S 3H 2C 7S 5H",
    "9D 8H 2C 6S 7H",
    "2D 6D 9D TH 7D",
    "TC 8C 2S JH 6C",
    "JH 8S TH AH QH",
    "TS KS 5S 9S AC",
    "KD 6S 9D TH AD",
    "KS 8D 4D 9S 4S",  # pair
    "8C 4S KH JS 4D",  # pair
    "QH 8H KD JH 8S",  # pair
    "KC 4H KS 2H 8D",  # pair
    "KD 4S KC 3H 8S",  # pair
    "AH 8S AS KC JH",  # pair
    "3H 4C 4H 3S 2H",  # 2 pairs
    "5S 5D 2C KH KH",  # 2 pairs
    "3C KH 5D 5S KH",  # 2 pairs
    "AS 3C KH AD KH",  # 2 pairs
    "7C 7S 3S 7H 5S",  # 3 of a kind
    "7C 7S KH 2H 7H",  # 3 of a kind
    "AC KH QH AH AS",  # 3 of a kind
    "2H 4D 3C AS 5S",  # straight (low ace)
    "3C 5C 4C 2C 6H",  # straight
    "6S 8S 7S 5H 9H",  # straight
    "JS QS 9H TS KH",  # straight
    "QC KH TS JS AH",  # straight (high ace)
    "8C 9C 5C 3C TC",  # flush
    "3S 8S 9S 5S KS",  # flush
    "4C 5C 9C 8C KC",  # flush
    "JH 8H AH KH QH",  # flush
    "3D 2H 3H 2C 2D",  # full house
    "2H 2C 3S 3H 3D",  # full house
    "KH KC 3S 3H 3D",  # full house
    "JC 6H JS JD JH",  # 4 of a kind
    "JC 7H JS JD JH",  # 4 of a kind
    "JC KH JS JD JH",  # 4 of a kind
    "2S AS 4S 5S 3S",  # straight flush (low ace)
    "2D 6D 3D 4D 5D",  # straight flush
    "5C 6C 3C 7C 4C",  # straight flush
    "JH 9H TH KH QH",  # straight flush
    "JH AH TH KH QH",  # royal flush (high ace straight flush)
)

_lowerCamelCase	:       Union[str, Any]      	=	(
    ("2H 3H 4H 5H 6H", "KS AS TS QS JS", "Loss"),
    ("2H 3H 4H 5H 6H", "AS AD AC AH JD", "Win"),
    ("AS AH 2H AD AC", "JS JD JC JH 3D", "Win"),
    ("2S AH 2H AS AC", "JS JD JC JH AD", "Loss"),
    ("2S AH 2H AS AC", "2H 3H 5H 6H 7H", "Win"),
    ("AS 3S 4S 8S 2S", "2H 3H 5H 6H 7H", "Win"),
    ("2H 3H 5H 6H 7H", "2S 3H 4H 5S 6C", "Win"),
    ("2S 3H 4H 5S 6C", "3D 4C 5H 6H 2S", "Tie"),
    ("2S 3H 4H 5S 6C", "AH AC 5H 6H AS", "Win"),
    ("2S 2H 4H 5S 4C", "AH AC 5H 6H AS", "Loss"),
    ("2S 2H 4H 5S 4C", "AH AC 5H 6H 7S", "Win"),
    ("6S AD 7H 4S AS", "AH AC 5H 6H 7S", "Loss"),
    ("2S AH 4H 5S KC", "AH AC 5H 6H 7S", "Loss"),
    ("2S 3H 6H 7S 9C", "7H 3C TH 6H 9S", "Loss"),
    ("4S 5H 6H TS AC", "3S 5H 6H TS AC", "Win"),
    ("2S AH 4H 5S 6C", "AD 4C 5H 6H 2C", "Tie"),
    ("AS AH 3H AD AC", "AS AH 2H AD AC", "Win"),
    ("AH AC 5H 5C QS", "AH AC 5H 5C KS", "Loss"),
    ("AH AC 5H 5C QS", "KH KC 5H 5C QS", "Win"),
    ("7C 7S KH 2H 7H", "3C 3S AH 2H 3H", "Win"),
    ("3C 3S AH 2H 3H", "7C 7S KH 2H 7H", "Loss"),
    ("6H 5H 4H 3H 2H", "5H 4H 3H 2H AH", "Win"),
    ("5H 4H 3H 2H AH", "5H 4H 3H 2H AH", "Tie"),
    ("5H 4H 3H 2H AH", "6H 5H 4H 3H 2H", "Loss"),
    ("AH AD KS KC AC", "AH KD KH AC KC", "Win"),
    ("2H 4D 3C AS 5S", "2H 4D 3C 6S 5S", "Loss"),
    ("2H 3S 3C 3H 2S", "3S 3C 2S 2H 2D", "Win"),
    ("4D 6D 5D 2D JH", "3S 8S 3H TC KH", "Loss"),
    ("4S 6C 8S 3S 7S", "AD KS 2D 7D 7C", "Loss"),
    ("6S 4C 7H 8C 3H", "5H JC AH 9D 9C", "Loss"),
    ("9D 9H JH TC QH", "3C 2S JS 5C 7H", "Win"),
    ("2H TC 8S AD 9S", "4H TS 7H 2C 5C", "Win"),
    ("9D 3S 2C 7S 7C", "JC TD 3C TC 9H", "Loss"),
)

_lowerCamelCase	:       Dict      	=	(
    ("2H 3H 4H 5H 6H", True),
    ("AS AH 2H AD AC", False),
    ("2H 3H 5H 6H 7H", True),
    ("KS AS TS QS JS", True),
    ("8H 9H QS JS TH", False),
    ("AS 3S 4S 8S 2S", True),
)

_lowerCamelCase	:       Dict      	=	(
    ("2H 3H 4H 5H 6H", True),
    ("AS AH 2H AD AC", False),
    ("2H 3H 5H 6H 7H", False),
    ("KS AS TS QS JS", True),
    ("8H 9H QS JS TH", True),
)

_lowerCamelCase	:       Optional[Any]      	=	(
    ("2H 4D 3C AS 5S", True, [5, 4, 3, 2, 14]),
    ("2H 5D 3C AS 5S", False, [14, 5, 5, 3, 2]),
    ("JH QD KC AS TS", False, [14, 13, 12, 11, 10]),
    ("9D 3S 2C 7S 7C", False, [9, 7, 7, 3, 2]),
)

_lowerCamelCase	:       List[Any]      	=	(
    ("JH AH TH KH QH", 0),
    ("JH 9H TH KH QH", 0),
    ("JC KH JS JD JH", 7),
    ("KH KC 3S 3H 3D", 6),
    ("8C 9C 5C 3C TC", 0),
    ("JS QS 9H TS KH", 0),
    ("7C 7S KH 2H 7H", 3),
    ("3C KH 5D 5S KH", 2),
    ("QH 8H KD JH 8S", 1),
    ("2D 6D 9D TH 7D", 0),
)

_lowerCamelCase	:       List[str]      	=	(
    ("JH AH TH KH QH", 23),
    ("JH 9H TH KH QH", 22),
    ("JC KH JS JD JH", 21),
    ("KH KC 3S 3H 3D", 20),
    ("8C 9C 5C 3C TC", 19),
    ("JS QS 9H TS KH", 18),
    ("7C 7S KH 2H 7H", 17),
    ("3C KH 5D 5S KH", 16),
    ("QH 8H KD JH 8S", 15),
    ("2D 6D 9D TH 7D", 14),
)







def 				__lowerCamelCase				(   )       ->	Optional[Any]:






   """simple docstring"""
   UpperCamelCase				,      UpperCamelCase					=  randrange(len(A__    )    ), randrange(len(A__    )    )
   UpperCamelCase					=  ['Loss', 'Tie', 'Win'][(play >= oppo) + (play > oppo)]
   UpperCamelCase				,      UpperCamelCase					=  SORTED_HANDS[play], SORTED_HANDS[oppo]
   return hand, other, expected







def 				__lowerCamelCase				(   A__ = 100    )       ->	Optional[Any]:






   """simple docstring"""
   return (generate_random_hand() for _ in range(A__    ))







@pytest.mark.parametrize('hand, expected'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    )       ->	Any:






   """simple docstring"""
   assert PokerHand(A__    )._is_flush() == expected







@pytest.mark.parametrize('hand, expected'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    )       ->	Any:






   """simple docstring"""
   assert PokerHand(A__    )._is_straight() == expected







@pytest.mark.parametrize('hand, expected, card_values'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    )       ->	str:






   """simple docstring"""
   UpperCamelCase					=  PokerHand(A__    )
   assert player._is_five_high_straight() == expected
   assert player._card_values == card_values







@pytest.mark.parametrize('hand, expected'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    )       ->	Dict:






   """simple docstring"""
   assert PokerHand(A__    )._is_same_kind() == expected







@pytest.mark.parametrize('hand, expected'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    )       ->	str:






   """simple docstring"""
   assert PokerHand(A__    )._hand_type == expected







@pytest.mark.parametrize('hand, other, expected'    ,						A__    )
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    )       ->	Tuple:






   """simple docstring"""
   assert PokerHand(A__    ).compare_with(PokerHand(A__    )    ) == expected







@pytest.mark.parametrize('hand, other, expected'    ,						generate_random_hands()    )
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    )       ->	List[str]:






   """simple docstring"""
   assert PokerHand(A__    ).compare_with(PokerHand(A__    )    ) == expected







def 				__lowerCamelCase				(   )       ->	str:






   """simple docstring"""
   UpperCamelCase					=  [PokerHand(A__    ) for hand in SORTED_HANDS]
   UpperCamelCase					=  poker_hands.copy()
   shuffle(A__    )
   UpperCamelCase					=  chain(sorted(A__    )    )
   for index, hand in enumerate(A__    ):
      assert hand == poker_hands[index]







def 				__lowerCamelCase				(   )       ->	Optional[int]:






   """simple docstring"""
   # Test that five high straights are compared correctly.
   UpperCamelCase					=  [PokerHand('2D AC 3H 4H 5S'    ), PokerHand('2S 3H 4H 5S 6C'    )]
   pokerhands.sort(reverse=A__    )
   assert pokerhands[0].__str__() == "2S 3H 4H 5S 6C"







def 				__lowerCamelCase				(   )       ->	str:






   """simple docstring"""
   # Multiple calls to five_high_straight function should still return True
   # and shouldn't mutate the list in every call other than the first.
   UpperCamelCase					=  PokerHand('2C 4S AS 3D 5C'    )
   UpperCamelCase					=  True
   UpperCamelCase					=  [5, 4, 3, 2, 14]
   for _ in range(10    ):
      assert pokerhand._is_five_high_straight() == expected
      assert pokerhand._card_values == expected_card_values







def 				__lowerCamelCase				(   )       ->	List[str]:






   """simple docstring"""
   # Problem number 54 from Project Euler
   # Testing from poker_hands.txt file
   UpperCamelCase					=  0
   UpperCamelCase					=  os.path.abspath(os.path.dirname(A__    )    )
   UpperCamelCase					=  os.path.join(A__    ,						'poker_hands.txt'    )
   with open(A__    ) as file_hand:
      for line in file_hand:
         UpperCamelCase					=  line[:14].strip()
         UpperCamelCase					=  line[15:].strip()
         UpperCamelCase				,      UpperCamelCase					=  PokerHand(A__    ), PokerHand(A__    )
         UpperCamelCase					=  player.compare_with(A__    )
         if output == "Win":
            answer += 1
   assert answer == 376

