
'''simple docstring'''







import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase :  str    							=       logging.get_logger(__name__)

_lowercase :  str    							=       {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_lowercase :  Dict    							=       {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

_lowercase :  Union[str, Any]    							=       {
    "ctrl": 2_5_6,
}

_lowercase :  List[Any]    							=       {
    "Pregnancy": 1_6_8_6_2_9,
    "Christianity": 7_6_7_5,
    "Explain": 1_0_6_4_2_3,
    "Fitness": 6_3_4_4_0,
    "Saving": 6_3_1_6_3,
    "Ask": 2_7_1_7_1,
    "Ass": 9_5_9_8_5,
    "Joke": 1_6_3_5_0_9,
    "Questions": 4_5_6_2_2,
    "Thoughts": 4_9_6_0_5,
    "Retail": 5_2_3_4_2,
    "Feminism": 1_6_4_3_3_8,
    "Writing": 1_1_9_9_2,
    "Atheism": 1_9_2_2_6_3,
    "Netflix": 4_8_6_1_6,
    "Computing": 3_9_6_3_9,
    "Opinion": 4_3_2_1_3,
    "Alone": 4_4_9_6_7,
    "Funny": 5_8_9_1_7,
    "Gaming": 4_0_3_5_8,
    "Human": 4_0_8_8,
    "India": 1_3_3_1,
    "Joker": 7_7_1_3_8,
    "Diet": 3_6_2_0_6,
    "Legal": 1_1_8_5_9,
    "Norman": 4_9_3_9,
    "Tip": 7_2_6_8_9,
    "Weight": 5_2_3_4_3,
    "Movies": 4_6_2_7_3,
    "Running": 2_3_4_2_5,
    "Science": 2_0_9_0,
    "Horror": 3_7_7_9_3,
    "Confession": 6_0_5_7_2,
    "Finance": 1_2_2_5_0,
    "Politics": 1_6_3_6_0,
    "Scary": 1_9_1_9_8_5,
    "Support": 1_2_6_5_4,
    "Technologies": 3_2_5_1_6,
    "Teenage": 6_6_1_6_0,
    "Event": 3_2_7_6_9,
    "Learned": 6_7_4_6_0,
    "Notion": 1_8_2_7_7_0,
    "Wikipedia": 3_7_5_8_3,
    "Books": 6_6_6_5,
    "Extract": 7_6_0_5_0,
    "Confessions": 1_0_2_7_0_1,
    "Conspiracy": 7_5_9_3_2,
    "Links": 6_3_6_7_4,
    "Narcissus": 1_5_0_4_2_5,
    "Relationship": 5_4_7_6_6,
    "Relationships": 1_3_4_7_9_6,
    "Reviews": 4_1_6_7_1,
    "News": 4_2_5_6,
    "Translation": 2_6_8_2_0,
    "multilingual": 1_2_8_4_0_6,
}
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Dict							):



						"""simple docstring"""





						lowercase_	:     List[str]													=       set()
						lowercase_	:     Tuple													=       word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)							)
												lowercase_	:     List[Any]													=       char

						lowercase_	:     Union[str, Any]													=       set(__SCREAMING_SNAKE_CASE							)
						return pairs






class  lowerCAmelCase__   (  lowerCamelCase_     ):
					lowerCAmelCase_        =     VOCAB_FILES_NAMES
					lowerCAmelCase_        =     PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_        =     CONTROL_CODES






					def __init__(   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE="<unk>"				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											super().__init__(unk_token=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)

											with open(__SCREAMING_SNAKE_CASE				,							encoding='''utf-8'''		) as vocab_handle:
																	lowercase_	:     int													=       json.load(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[Any]													=       {v: k for k, v in self.encoder.items()}
											with open(__SCREAMING_SNAKE_CASE				,							encoding='''utf-8'''		) as merges_handle:
																	lowercase_	:     Optional[Any]													=       merges_handle.read().split('''\n'''		)[1:-1]
											lowercase_	:     Tuple													=       [tuple(merge.split()		) for merge in merges]
											lowercase_	:     Union[str, Any]													=       dict(zip(__SCREAMING_SNAKE_CASE				,							range(len(__SCREAMING_SNAKE_CASE		)		)		)		)
											lowercase_	:     str													=       {}






					@property
					def 	_snake_case    (   self		):


											"""simple docstring"""





											return len(self.encoder		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											return dict(self.encoder				,							**self.added_tokens_encoder		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.cache:
																	return self.cache[token]
											lowercase_	:     Tuple													=       tuple(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[Any]													=       tuple(list(word[:-1]		) + [word[-1] + '''</w>''']		)
											lowercase_	:     Any													=       get_pairs(__SCREAMING_SNAKE_CASE		)

											if not pairs:
																	return token

											while True:
																	lowercase_	:     Optional[Any]													=       min(__SCREAMING_SNAKE_CASE				,							key=lambda __SCREAMING_SNAKE_CASE		: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE				,							float('''inf'''		)		)		)
																	if bigram not in self.bpe_ranks:
																							break
																	lowercase_						, lowercase_	:     str													=       bigram
																	lowercase_	:     List[str]													=       []
																	lowercase_	:     Tuple													=       0
																	while i < len(__SCREAMING_SNAKE_CASE		):
																							try:
																													lowercase_	:     int													=       word.index(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)
																							except ValueError:
																													new_word.extend(word[i:]		)
																													break
																							else:
																													new_word.extend(word[i:j]		)
																													lowercase_	:     int													=       j

																							if word[i] == first and i < len(__SCREAMING_SNAKE_CASE		) - 1 and word[i + 1] == second:
																													new_word.append(first + second		)
																													i += 2
																							else:
																													new_word.append(word[i]		)
																													i += 1
																	lowercase_	:     List[Any]													=       tuple(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     List[str]													=       new_word
																	if len(__SCREAMING_SNAKE_CASE		) == 1:
																							break
																	else:
																							lowercase_	:     Tuple													=       get_pairs(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Dict													=       '''@@ '''.join(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Optional[Any]													=       word[:-4]
											lowercase_	:     Union[str, Any]													=       word
											return word






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Union[str, Any]													=       []

											lowercase_	:     str													=       re.findall(R'''\S+\n?'''				,							__SCREAMING_SNAKE_CASE		)

											for token in words:
																	split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE		).split(''' '''		)		)		)
											return split_tokens






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.encoder.get(__SCREAMING_SNAKE_CASE				,							self.encoder.get(self.unk_token		)		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.decoder.get(__SCREAMING_SNAKE_CASE				,							self.unk_token		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       ''' '''.join(__SCREAMING_SNAKE_CASE		).replace('''@@ '''				,							''''''		).strip()
											return out_string






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_	:     Tuple													=       os.path.join(
											    __SCREAMING_SNAKE_CASE				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
											lowercase_	:     int													=       os.path.join(
											    __SCREAMING_SNAKE_CASE				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

											with open(__SCREAMING_SNAKE_CASE				,							'''w'''				,							encoding='''utf-8'''		) as f:
																	f.write(json.dumps(self.encoder				,							indent=2				,							sort_keys=__SCREAMING_SNAKE_CASE				,							ensure_ascii=__SCREAMING_SNAKE_CASE		) + '''\n'''		)

											lowercase_	:     Dict													=       0
											with open(__SCREAMING_SNAKE_CASE				,							'''w'''				,							encoding='''utf-8'''		) as writer:
																	writer.write('''#version: 0.2\n'''		)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,							key=lambda __SCREAMING_SNAKE_CASE		: kv[1]		):
																							if index != token_index:
																													logger.warning(
																													    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    ''' Please check that the tokenizer is not corrupted!'''		)
																													lowercase_	:     str													=       token_index
																							writer.write(''' '''.join(__SCREAMING_SNAKE_CASE		) + '''\n'''		)
																							index += 1

											return vocab_file, merge_file

					# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
					#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
					#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
					#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
					#     return ''.join(tokens_generated_so_far)











def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if not head:
										return True
					# split the list to two parts
					UpperCAmelCase_       ,  UpperCAmelCase_    :      Any            =						head.next, head
					while fast and fast.next:
										UpperCAmelCase_    :      str            =						fast.next.next
										UpperCAmelCase_    :      Union[str, Any]            =						slow.next
					UpperCAmelCase_    :      int            =						slow.next
					UpperCAmelCase_    :      List[Any]            =						None  # Don't forget here! But forget still works!
					# reverse the second part
					UpperCAmelCase_    :      Tuple            =						None
					while second:
										UpperCAmelCase_    :      int            =						second.next
										UpperCAmelCase_    :      Any            =						node
										UpperCAmelCase_    :      Optional[Any]            =						second
										UpperCAmelCase_    :      Tuple            =						nxt
					# compare two parts
					# second part has the same or one less node
					while node:
										if node.val != head.val:
															return False
										UpperCAmelCase_    :      Optional[Any]            =						node.next
										UpperCAmelCase_    :      Dict            =						head.next
					return True




def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					if not head or not head.next:
										return True

					# 1. Get the midpoint (slow)
					UpperCAmelCase_    :      Any            =						head
					while fast and fast.next:
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Tuple            =						fast.next.next, slow.next

					# 2. Push the second half into the stack
					UpperCAmelCase_    :      List[str]            =						[slow.val]
					while slow.next:
										UpperCAmelCase_    :      List[str]            =						slow.next
										stack.append(slow.val  )

					# 3. Comparison
					while stack:
										if stack.pop() != cur.val:
															return False
										UpperCAmelCase_    :      int            =						cur.next

					return True






def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if not head or not head.next:
										return True
					UpperCAmelCase_    :      Tuple            =						{}
					UpperCAmelCase_    :      int            =						0
					while head:
										if head.val in d:
															d[head.val].append(__snake_case  )
										else:
															UpperCAmelCase_    :      List[Any]            =						[pos]
										UpperCAmelCase_    :      Any            =						head.next
										pos += 1
					UpperCAmelCase_    :      Dict            =						pos - 1
					UpperCAmelCase_    :      Optional[int]            =						0
					for v in d.values():
										if len(__snake_case  ) % 2 != 0:
															middle += 1
										else:
															UpperCAmelCase_    :      int            =						0
															for i in range(0     ,       len(__snake_case  )  ):
																				if v[i] + v[len(__snake_case  ) - 1 - step] != checksum:
																									return False
																				step += 1
										if middle > 1:
															return False
					return True






