



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_lowercase:					int      =						logging.get_logger(__name__)

_lowercase:					int      =						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_lowercase:					Dict      =						{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

_lowercase:					Any      =						{
    "Salesforce/codegen-350M-mono": 2048,
}






@lru_cache()
def        a( )						->			List[str]:
					"""simple docstring"""





					a   	=  (
					    list(range(ord("!"			)					,	ord("~"			) + 1			)			) + list(range(ord("¡"			)					,	ord("¬"			) + 1			)			) + list(range(ord("®"			)					,	ord("ÿ"			) + 1			)			)
					)
					a   	=  bs[:]
					a   	=  0
					for b in range(2**8			):
										if b not in bs:
															bs.append(lowerCAmelCase_			)
															cs.append(2**8 + n			)
															n += 1
					a   	=  [chr(lowerCAmelCase_			) for n in cs]
					return dict(zip(lowerCAmelCase_					,	lowerCAmelCase_			)			)






def        a( A			:       int			)						->			Any:
					"""simple docstring"""





					a   	=  set()
					a   	=  word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)			)
										a   	=  char
					return pairs







class       _lowercase				(  snake_case__      ):





	"""simple docstring"""
	__A		      =				VOCAB_FILES_NAMES
	__A		      =				PRETRAINED_VOCAB_FILES_MAP
	__A		      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A		      =				['input_ids', 'attention_mask']




	def __init__(self						,			lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_="replace"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_=None						,			lowerCamelCase_=False						,			lowerCamelCase_=False						,			**lowerCamelCase_						,			):
						"""simple docstring"""

						a   	=  AddedToken(lowerCAmelCase_						,			lstrip=lowerCAmelCase_						,			rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_						,			lowerCAmelCase_						) else bos_token
						a   	=  AddedToken(lowerCAmelCase_						,			lstrip=lowerCAmelCase_						,			rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_						,			lowerCAmelCase_						) else eos_token
						a   	=  AddedToken(lowerCAmelCase_						,			lstrip=lowerCAmelCase_						,			rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_						,			lowerCAmelCase_						) else unk_token
						a   	=  AddedToken(lowerCAmelCase_						,			lstrip=lowerCAmelCase_						,			rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_						,			lowerCAmelCase_						) else pad_token
						super().__init__(
						    errors=lowerCAmelCase_						,			unk_token=lowerCAmelCase_						,			bos_token=lowerCAmelCase_						,			eos_token=lowerCAmelCase_						,			pad_token=lowerCAmelCase_						,			add_prefix_space=lowerCAmelCase_						,			add_bos_token=lowerCAmelCase_						,			**lowerCAmelCase_						,			)
						a   	=  add_bos_token

						with open(lowerCAmelCase_						,			encoding="utf-8"						) as vocab_handle:
											a   	=  json.load(lowerCAmelCase_						)
						a   	=  {v: k for k, v in self.encoder.items()}
						a   	=  errors  # how to handle errors in decoding
						a   	=  bytes_to_unicode()
						a   	=  {v: k for k, v in self.byte_encoder.items()}
						with open(lowerCAmelCase_						,			encoding="utf-8"						) as merges_handle:
											a   	=  merges_handle.read().split("\n"						)[1:-1]
						a   	=  [tuple(merge.split()						) for merge in bpe_merges]
						a   	=  dict(zip(lowerCAmelCase_						,			range(len(lowerCAmelCase_						)						)						)						)
						a   	=  {}
						a   	=  add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						a   	=  re.compile(R"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"						)




	@property
	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return len(self.encoder						)




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return dict(self.encoder						,			**self.added_tokens_encoder						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if token in self.cache:
											return self.cache[token]
						a   	=  tuple(lowerCAmelCase_						)
						a   	=  get_pairs(lowerCAmelCase_						)

						if not pairs:
											return token

						while True:
											a   	=  min(lowerCAmelCase_						,			key=lambda lowerCamelCase_						: self.bpe_ranks.get(lowerCAmelCase_						,			float("inf"						)						)						)
											if bigram not in self.bpe_ranks:
																break
											a   	=  bigram
											a   	=  []
											a   	=  0
											while i < len(lowerCAmelCase_						):
																try:
																					a   	=  word.index(lowerCAmelCase_						,			lowerCAmelCase_						)
																except ValueError:
																					new_word.extend(word[i:]						)
																					break
																else:
																					new_word.extend(word[i:j]						)
																					a   	=  j

																if word[i] == first and i < len(lowerCAmelCase_						) - 1 and word[i + 1] == second:
																					new_word.append(first + second						)
																					i += 2
																else:
																					new_word.append(word[i]						)
																					i += 1
											a   	=  tuple(lowerCAmelCase_						)
											a   	=  new_word
											if len(lowerCAmelCase_						) == 1:
																break
											else:
																a   	=  get_pairs(lowerCAmelCase_						)
						a   	=  ''' '''.join(lowerCAmelCase_						)
						a   	=  word
						return word




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_=None						):
						"""simple docstring"""

						if self.add_bos_token:
											a   	=  [self.bos_token_id]
						else:
											a   	=  []

						a   	=  bos_token_ids + token_ids_a

						if token_ids_a is None:
											return output

						return output + bos_token_ids + token_ids_a




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  []
						for token in re.findall(self.pat						,			lowerCAmelCase_						):
											a   	=  ''''''.join(
											    self.byte_encoder[b] for b in token.encode("utf-8"						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase_						).split(" "						)						)
						return bpe_tokens




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return self.encoder.get(lowerCAmelCase_						,			self.encoder.get(self.unk_token						)						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return self.decoder.get(lowerCAmelCase_						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  ''''''.join(lowerCAmelCase_						)
						a   	=  bytearray([self.byte_decoder[c] for c in text]						).decode("utf-8"						,			errors=self.errors						)
						return text




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						):
						"""simple docstring"""

						if not os.path.isdir(lowerCAmelCase_						):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
						a   	=  os.path.join(
						    lowerCAmelCase_						,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
						a   	=  os.path.join(
						    lowerCAmelCase_						,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

						with open(lowerCAmelCase_						,			"w"						,			encoding="utf-8"						) as f:
											f.write(json.dumps(self.encoder						,			indent=2						,			sort_keys=lowerCAmelCase_						,			ensure_ascii=lowerCAmelCase_						) + "\n"						)

						a   	=  0
						with open(lowerCAmelCase_						,			"w"						,			encoding="utf-8"						) as writer:
											writer.write("#version: 0.2\n"						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,			key=lambda lowerCamelCase_						: kv[1]						):
																if index != token_index:
																					logger.warning(
																					    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																					    " Please check that the tokenizer is not corrupted!"						)
																					a   	=  token_index
																writer.write(" ".join(lowerCAmelCase_						) + "\n"						)
																index += 1

						return vocab_file, merge_file




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_=False						,			**lowerCamelCase_						):
						"""simple docstring"""

						a   	=  kwargs.pop("add_prefix_space"						,			self.add_prefix_space						)
						if is_split_into_words or add_prefix_space:
											a   	=  ''' ''' + text
						return (text, kwargs)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = False						,			lowerCamelCase_ = None						,			lowerCamelCase_ = None						,			**lowerCamelCase_						,			):
						"""simple docstring"""

						a   	=  super()._decode(
						    token_ids=lowerCAmelCase_						,			skip_special_tokens=lowerCAmelCase_						,			clean_up_tokenization_spaces=lowerCAmelCase_						,			**lowerCAmelCase_						,			)

						if truncate_before_pattern is not None and len(lowerCAmelCase_						) > 0:
											a   	=  self.truncate(lowerCAmelCase_						,			lowerCAmelCase_						)

						return decoded_text




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_						):
						"""simple docstring"""

						def find_re(lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_						):
											a   	=  pattern.search(lowerCAmelCase_						,			lowerCAmelCase_						)
											return m.start() if m else -1

						a   	=  [re.compile(lowerCAmelCase_						,			re.MULTILINE						) for pattern in truncate_before_pattern]

						a   	=  list(re.finditer("^print"						,			lowerCAmelCase_						,			re.MULTILINE						)						)

						if len(lowerCAmelCase_						) > 1:
											a   	=  completion[: prints[1].start()]

						a   	=  list(re.finditer("^def"						,			lowerCAmelCase_						,			re.MULTILINE						)						)

						if len(lowerCAmelCase_						) > 1:
											a   	=  completion[: defs[1].start()]

						a   	=  0

						a   	=  [
						    pos for pos in [find_re(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						) for terminal in terminals] if pos != -1
						]

						if len(lowerCAmelCase_						) > 0:
											return completion[: min(lowerCAmelCase_						)]
						else:
											return completion

import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata







def 						UpperCAmelCase							(		lowerCAmelCase_							)						->						str:
     a	:     Optional[Any]						      =    min(re.search('''^ +'''    ,   lowerCAmelCase_							).end() if t.startswith(''' '''							) else 0 for t in string.splitlines()							)
     return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(lowerCAmelCase_							)]							)


UpperCAmelCase				 =   """\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
"""

UpperCAmelCase				 =   """\
---
---
# Begin of markdown

Some cool dataset card
"""


UpperCAmelCase				 =   """\
# Begin of markdown

Some cool dataset card
"""






class 				_SCREAMING_SNAKE_CASE( unittest.TestCase						):
   def _lowercase(     self     )						->List[str]:
        with tempfile.TemporaryDirectory() as tmp_dir:
             a	:     List[str]						      =    Path(lowerCAmelCase_     ) / '''README.md'''
             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     Dict						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertDictEqual(lowerCAmelCase_				,  {'''language''': ['''zh''', '''en'''], '''task_ids''': ['''sentiment-classification''']}     )

             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     List[str]						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertDictEqual(lowerCAmelCase_				,  {}     )

             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     List[Any]						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertEqual(lowerCAmelCase_				,  {}     )


   def _lowercase(     self     )						->Optional[Any]:
        a	:     Tuple						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            '''     )
        assert DatasetMetadata.from_yaml_string(lowerCAmelCase_     )

        a	:     Dict						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            task_ids:
            - open-domain-qa
            '''     )
        with self.assertRaises(lowerCAmelCase_     ):
             DatasetMetadata.from_yaml_string(lowerCAmelCase_     )

        a	:     Optional[int]						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - text-classification
            task_ids:
            - multi-class-classification
            paperswithcode_id:
            - squad
            configs:
            - en
            train-eval-index:
            - config: en
              task: text-classification
              task_id: multi_class_classification
              splits:
                train_split: train
                eval_split: test
              col_mapping:
                text: text
                label: target
              metrics:
                - type: accuracy
                  name: Accuracy
            extra_gated_prompt: |
              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:
              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:
              1. Researcher shall use the Database only for non-commercial research and educational purposes.
            extra_gated_fields:
              Company: text
              Country: text
              I agree to use this model for non-commerical use ONLY: checkbox
            '''     )
        assert DatasetMetadata.from_yaml_string(lowerCAmelCase_     )