"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_a	:	str=					logging.get_logger(__name__)

_a	:	int=					{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_a	:	Optional[Any]=					{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

_a	:	List[str]=					{
    """Salesforce/codegen-350M-mono""": 2_048,
}
@lru_cache()
def     __UpperCAmelCase							(       )				->     Tuple:

	'''simple docstring'''





	__snake_case     :				List[Any] 						=					(
	    list(range(ord('!'    )     ,				ord('~'    ) + 1    )    ) + list(range(ord('¡'    )     ,				ord('¬'    ) + 1    )    ) + list(range(ord('®'    )     ,				ord('ÿ'    ) + 1    )    )
	)
	__snake_case     :				Optional[Any] 						=					bs[:]
	__snake_case     :				List[str] 						=					0
	for b in range(2**8    ):
		if b not in bs:
			bs.append(UpperCAmelCase_    )
			cs.append(2**8 + n    )
			n += 1
	__snake_case     :				int 						=					[chr(UpperCAmelCase_    ) for n in cs]
	return dict(zip(UpperCAmelCase_     ,				UpperCAmelCase_    )    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							int    )				->     int:

	'''simple docstring'''





	__snake_case     :				Tuple 						=					set()
	__snake_case     :				List[str] 						=					word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)    )
		__snake_case     :				Optional[Any] 						=					char
	return pairs

class 						UpperCamelCase     (       _SCREAMING_SNAKE_CASE		):
				UpperCAmelCase							: List[Any]       =						VOCAB_FILES_NAMES
				UpperCAmelCase							: Dict       =						PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase							: str       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase							: str       =						["""input_ids""", """attention_mask"""]



				def __init__(self							: Dict      ,    _A							: int      ,    _A							: Union[str, Any]      ,    _A							: Any="replace"      ,    _A							: int="<|endoftext|>"      ,    _A							: List[str]="<|endoftext|>"      ,    _A							: List[str]="<|endoftext|>"      ,    _A							: Dict=None      ,    _A							: Optional[int]=False      ,    _A							: Dict=False      ,    **_A							: List[str]      ,    )       ->      int:
					__snake_case     :				int 						=					AddedToken(__A      ,    lstrip=__A      ,    rstrip=__A) if isinstance(__A      ,    __A) else bos_token
					__snake_case     :				List[Any] 						=					AddedToken(__A      ,    lstrip=__A      ,    rstrip=__A) if isinstance(__A      ,    __A) else eos_token
					__snake_case     :				Tuple 						=					AddedToken(__A      ,    lstrip=__A      ,    rstrip=__A) if isinstance(__A      ,    __A) else unk_token
					__snake_case     :				int 						=					AddedToken(__A      ,    lstrip=__A      ,    rstrip=__A) if isinstance(__A      ,    __A) else pad_token
					super().__init__(
					    errors=__A      ,    unk_token=__A      ,    bos_token=__A      ,    eos_token=__A      ,    pad_token=__A      ,    add_prefix_space=__A      ,    add_bos_token=__A      ,    **__A      ,    )
					__snake_case     :				List[str] 						=					add_bos_token

					with open(__A      ,    encoding='utf-8') as vocab_handle:
						__snake_case     :				Optional[int] 						=					json.load(__A)
					__snake_case     :				Dict 						=					{v: k for k, v in self.encoder.items()}
					__snake_case     :				Optional[Any] 						=					errors  # how to handle errors in decoding
					__snake_case     :				List[str] 						=					bytes_to_unicode()
					__snake_case     :				Optional[Any] 						=					{v: k for k, v in self.byte_encoder.items()}
					with open(__A      ,    encoding='utf-8') as merges_handle:
						__snake_case     :				int 						=					merges_handle.read().split('\n')[1:-1]
					__snake_case     :				List[str] 						=					[tuple(merge.split()) for merge in bpe_merges]
					__snake_case     :				Optional[Any] 						=					dict(zip(__A      ,    range(len(__A))))
					__snake_case     :				Tuple 						=					{}
					__snake_case     :				str 						=					add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__snake_case     :				Tuple 						=					re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')



				@property
				def    _lowercase	(self							: Dict)       ->      List[str]:
					return len(self.encoder)



				def    _lowercase	(self							: int)       ->      Optional[Any]:
					return dict(self.encoder      ,    **self.added_tokens_encoder)



				def    _lowercase	(self							: Optional[Any]      ,    _A							: Dict)       ->      List[str]:
					if token in self.cache:
						return self.cache[token]
					__snake_case     :				List[str] 						=					tuple(__A)
					__snake_case     :				Union[str, Any] 						=					get_pairs(__A)

					if not pairs:
						return token

					while True:
						__snake_case     :				Tuple 						=					min(__A      ,    key=lambda _A: self.bpe_ranks.get(__A      ,    float('inf')))
						if bigram not in self.bpe_ranks:
							break
						__snake_case						,							__snake_case     :				List[Any] 						=					bigram
						__snake_case     :				Tuple 						=					[]
						__snake_case     :				List[Any] 						=					0
						while i < len(__A):
							try:
								__snake_case     :				Union[str, Any] 						=					word.index(__A      ,    __A)
							except ValueError:
								new_word.extend(word[i:])
								break
							else:
								new_word.extend(word[i:j])
								__snake_case     :				List[str] 						=					j

							if word[i] == first and i < len(__A) - 1 and word[i + 1] == second:
								new_word.append(first + second)
								i += 2
							else:
								new_word.append(word[i])
								i += 1
						__snake_case     :				Any 						=					tuple(__A)
						__snake_case     :				Optional[int] 						=					new_word
						if len(__A) == 1:
							break
						else:
							__snake_case     :				List[Any] 						=					get_pairs(__A)
					__snake_case     :				Any 						=					' '.join(__A)
					__snake_case     :				Optional[int] 						=					word
					return word



				def    _lowercase	(self							: str      ,    _A							: int      ,    _A							: Dict=None)       ->      Optional[Any]:
					if self.add_bos_token:
						__snake_case     :				Optional[int] 						=					[self.bos_token_id]
					else:
						__snake_case     :				Tuple 						=					[]

					__snake_case     :				Tuple 						=					bos_token_ids + token_ids_a

					if token_ids_a is None:
						return output

					return output + bos_token_ids + token_ids_a



				def    _lowercase	(self							: Tuple      ,    _A							: Union[str, Any])       ->      List[str]:
					__snake_case     :				Optional[Any] 						=					[]
					for token in re.findall(self.pat      ,    __A):
						__snake_case     :				List[Any] 						=					''.join(
						    self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
						bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A).split(' '))
					return bpe_tokens



				def    _lowercase	(self							: List[str]      ,    _A							: Optional[Any])       ->      Optional[int]:
					return self.encoder.get(__A      ,    self.encoder.get(self.unk_token))



				def    _lowercase	(self							: Union[str, Any]      ,    _A							: List[Any])       ->      Any:
					return self.decoder.get(__A)



				def    _lowercase	(self							: int      ,    _A							: Any)       ->      List[Any]:
					__snake_case     :				int 						=					''.join(__A)
					__snake_case     :				Optional[int] 						=					bytearray([self.byte_decoder[c] for c in text]).decode('utf-8'      ,    errors=self.errors)
					return text



				def    _lowercase	(self							: List[Any]      ,    _A							: Optional[int]      ,    _A							: str = None)       ->      Tuple[str]:
					if not os.path.isdir(__A):
						logger.error(f"Vocabulary path ({save_directory}) should be a directory")
						return
					__snake_case     :				Optional[Any] 						=					os.path.join(
					    __A      ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
					__snake_case     :				int 						=					os.path.join(
					    __A      ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

					with open(__A      ,    'w'      ,    encoding='utf-8') as f:
						f.write(json.dumps(self.encoder      ,    indent=2      ,    sort_keys=__A      ,    ensure_ascii=__A) + '\n')

					__snake_case     :				Dict 						=					0
					with open(__A      ,    'w'      ,    encoding='utf-8') as writer:
						writer.write('#version: 0.2\n')
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,    key=lambda _A: kv[1]):
							if index != token_index:
								logger.warning(
								    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
								    ' Please check that the tokenizer is not corrupted!')
								__snake_case     :				Optional[int] 						=					token_index
							writer.write(' '.join(__A) + '\n')
							index += 1

					return vocab_file, merge_file



				def    _lowercase	(self							: Dict      ,    _A							: Dict      ,    _A							: Optional[int]=False      ,    **_A							: Dict)       ->      Optional[int]:
					__snake_case     :				Dict 						=					kwargs.pop('add_prefix_space'      ,    self.add_prefix_space)
					if is_split_into_words or add_prefix_space:
						__snake_case     :				Tuple 						=					' ' + text
					return (text, kwargs)



				def    _lowercase	(self							: int      ,    _A							: Dict      ,    _A							: str = False      ,    _A							: List[str] = None      ,    _A							: str = None      ,    **_A							: Any      ,    )       ->      str:
					__snake_case     :				List[str] 						=					super()._decode(
					    token_ids=__A      ,    skip_special_tokens=__A      ,    clean_up_tokenization_spaces=__A      ,    **__A      ,    )

					if truncate_before_pattern is not None and len(__A) > 0:
						__snake_case     :				str 						=					self.truncate(__A      ,    __A)

					return decoded_text






				def    _lowercase	(self							: Optional[Any]      ,    _A							: Optional[int]      ,    _A							: Tuple)       ->      List[Any]:
					def find_re(_A							: str      ,    _A							: Optional[Any]      ,    _A							: Any):
						__snake_case     :				str 						=					pattern.search(__A      ,    __A)
						return m.start() if m else -1

					__snake_case     :				Dict 						=					[re.compile(__A      ,    re.MULTILINE) for pattern in truncate_before_pattern]

					__snake_case     :				List[Any] 						=					list(re.finditer('^print'      ,    __A      ,    re.MULTILINE))

					if len(__A) > 1:
						__snake_case     :				Optional[int] 						=					completion[: prints[1].start()]

					__snake_case     :				str 						=					list(re.finditer('^def'      ,    __A      ,    re.MULTILINE))

					if len(__A) > 1:
						__snake_case     :				Tuple 						=					completion[: defs[1].start()]

					__snake_case     :				Union[str, Any] 						=					0

					__snake_case     :				List[Any] 						=					[
					    pos for pos in [find_re(__A      ,    __A      ,    __A) for terminal in terminals] if pos != -1
					]

					if len(__A) > 0:
						return completion[: min(__A)]
					else:
						return completion









"""simple docstring"""







import argparse
import json
import re
import string
import sys
from collections import Counter
def _A		(		lowercase						):







    """simple docstring"""

    def remove_articles(lowercase						):
        return re.sub(R'''\b(a|an|the)\b'''			,       ''' '''			,       lowercase						)

    def white_space_fix(lowercase						):
        return " ".join(text.split()						)

    def remove_punc(lowercase						):
        a											=set(string.punctuation						)
        return "".join(ch for ch in text if ch not in exclude						)

    def lower(lowercase						):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(lowercase						)						)						)						)
def _A		(		lowercase			,       lowercase						):







    """simple docstring"""
    a											=normalize_answer(lowercase						).split()
    a											=normalize_answer(lowercase						).split()
    a											=Counter(lowercase						) & Counter(lowercase						)
    a											=sum(common.values()						)
    if num_same == 0:
        return 0
    a											=1.0 * num_same / len(lowercase						)
    a											=1.0 * num_same / len(lowercase						)
    a											=(2 * precision * recall) / (precision + recall)
    return fa
def _A		(		lowercase			,       lowercase						):







    """simple docstring"""
    return normalize_answer(lowercase						) == normalize_answer(lowercase						)
def _A		(		lowercase			,       lowercase			,       lowercase						):







    """simple docstring"""
    a											=[]
    for ground_truth in ground_truths:
        a											=metric_fn(lowercase			,       lowercase						)
        scores_for_ground_truths.append(lowercase						)
    return max(lowercase						)
def _A		(		lowercase			,       lowercase						):







    """simple docstring"""
    a											=a											=a											=0
    for article in dataset:
        for paragraph in article["paragraphs"]:
            for qa in paragraph["qas"]:
                total += 1
                if qa["id"] not in predictions:
                    a											='''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
                    print(lowercase			,       file=sys.stderr						)
                    continue
                a											=[x['''text'''] for x in qa['''answers''']]
                a											=predictions[qa['''id''']]
                exact_match += metric_max_over_ground_truths(lowercase			,       lowercase			,       lowercase						)
                fa += metric_max_over_ground_truths(lowercase			,       lowercase			,       lowercase						)

    a											=100.0 * exact_match / total
    a											=100.0 * fa / total

    return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
       lowerCamelCase_  :  List[Any]     		=    """1.1"""
       lowerCamelCase_  :  Optional[Any]     		=    argparse.ArgumentParser(description="""Evaluation for SQuAD """ + expected_version)
       parser.add_argument("""dataset_file""", help="""Dataset file""")
       parser.add_argument("""prediction_file""", help="""Prediction File""")
       lowerCamelCase_  :  Optional[Any]     		=    parser.parse_args()
       with open(args.dataset_file) as dataset_file:
              lowerCamelCase_  :  List[str]     		=    json.load(dataset_file)
              if dataset_json["version"] != expected_version:
                     print(
                         """Evaluation expects v-""" + expected_version + """, but got dataset with v-""" + dataset_json["""version"""],
                         file=sys.stderr,
                     )
              lowerCamelCase_  :  Dict     		=    dataset_json["""data"""]
       with open(args.prediction_file) as prediction_file:
              lowerCamelCase_  :  Optional[Any]     		=    json.load(prediction_file)
       print(json.dumps(evaluate(dataset, predictions)))