



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ast
import difflib
from collections.abc import Mapping
from typing import Any, Callable, Dict



class 		_lowercase		(							UpperCAmelCase__						):
   pass


def 						UpperCAmelCase__   (   UpperCAmelCase__			:str			,       UpperCAmelCase__			:Dict[str, Callable]			,       UpperCAmelCase__			:Optional[Any]=None			,       UpperCAmelCase__			:Optional[int]=False       ):






 '''simple docstring'''

 try:
  a							=		ast.parse(UpperCAmelCase__       )
 except SyntaxError as e:
  print("The code generated by the agent is not valid.\n"			,       UpperCAmelCase__       )
  return
 if state is None:
  a							=		{}
 a							=		None
 for idx, node in enumerate(expression.body       ):
  try:
   a							=		evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
  except InterpretorError as e:
   a							=		F"""Evaluation of the code stopped at line {idx} before the end because of the following error"""
   if chat_mode:
    msg += (
        F""". Copy paste the following error message and send it back to the agent:\nI get an error: '{e}'"""
    )
   else:
    msg += F""":\n{e}"""
   print(UpperCAmelCase__       )
   break
  if line_result is not None:
   a							=		line_result

 return result


def 						UpperCAmelCase__   (   UpperCAmelCase__			:ast.AST			,       UpperCAmelCase__			:Dict[str, Any]			,       UpperCAmelCase__			:Dict[str, Callable]       ):






 '''simple docstring'''

 if isinstance(UpperCAmelCase__			,       ast.Assign       ):
  # Assignement -> we evaluate the assignement which should update the state
  # We return the variable assigned as it may be used to determine the final result.
  return evaluate_assign(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.Call       ):
  # Function call -> we return the value of the function call
  return evaluate_call(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.Constant       ):
  # Constant -> just return the value
  return expression.value
 elif isinstance(UpperCAmelCase__			,       ast.Dict       ):
  # Dict -> evaluate all keys and values
  a							=		[evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       ) for k in expression.keys]
  a							=		[evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       ) for v in expression.values]
  return dict(zip(UpperCAmelCase__			,       UpperCAmelCase__       )       )
 elif isinstance(UpperCAmelCase__			,       ast.Expr       ):
  # Expression -> evaluate the content
  return evaluate_ast(expression.value			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.For       ):
  # For loop -> execute the loop
  return evaluate_for(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.FormattedValue       ):
  # Formatted value (part of f-string) -> evaluate the content and return
  return evaluate_ast(expression.value			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.If       ):
  # If -> execute the right branch
  return evaluate_if(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif hasattr(UpperCAmelCase__			,       "Index"       ) and isinstance(UpperCAmelCase__			,       ast.Index       ):
  return evaluate_ast(expression.value			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.JoinedStr       ):
  return "".join([str(evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )       ) for v in expression.values]       )
 elif isinstance(UpperCAmelCase__			,       ast.List       ):
  # List -> evaluate all elements
  return [evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       ) for elt in expression.elts]
 elif isinstance(UpperCAmelCase__			,       ast.Name       ):
  # Name -> pick up the value in the state
  return evaluate_name(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 elif isinstance(UpperCAmelCase__			,       ast.Subscript       ):
  # Subscript -> return the value of the indexing
  return evaluate_subscript(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 else:
  # For now we refuse anything else. Let's add things as we need them.
  raise InterpretorError(F"""{expression.__class__.__name__} is not supported."""       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:int			,       UpperCAmelCase__			:Dict			,       UpperCAmelCase__			:Tuple       ):






 '''simple docstring'''

 a							=		assign.targets
 a							=		evaluate_ast(assign.value			,       UpperCAmelCase__			,       UpperCAmelCase__       )

 if len(UpperCAmelCase__       ) == 1:
  a							=		result
 else:
  if len(UpperCAmelCase__       ) != len(UpperCAmelCase__       ):
   raise InterpretorError(F"""Expected {len(UpperCAmelCase__       )} values but got {len(UpperCAmelCase__       )}."""       )
  for var_name, r in zip(UpperCAmelCase__			,       UpperCAmelCase__       ):
   a							=		r
 return result


def 						UpperCAmelCase__   (   UpperCAmelCase__			:int			,       UpperCAmelCase__			:str			,       UpperCAmelCase__			:Tuple       ):






 '''simple docstring'''

 if not isinstance(call.func			,       ast.Name       ):
  raise InterpretorError(
      F"""It is not permitted to evaluate other functions than the provided tools (tried to execute {call.func} of """
      F"""type {type(call.func       )}."""       )
 a							=		call.func.id
 if func_name not in tools:
  raise InterpretorError(
      F"""It is not permitted to evaluate other functions than the provided tools (tried to execute {call.func.id})."""       )

 a							=		tools[func_name]
 # Todo deal with args
 a							=		[evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       ) for arg in call.args]
 a							=		{keyword.arg: evaluate_ast(keyword.value			,       UpperCAmelCase__			,       UpperCAmelCase__       ) for keyword in call.keywords}
 return func(*UpperCAmelCase__			,       **UpperCAmelCase__       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:str			,       UpperCAmelCase__			:List[str]			,       UpperCAmelCase__			:Union[str, Any]       ):






 '''simple docstring'''

 a							=		evaluate_ast(subscript.slice			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 a							=		evaluate_ast(subscript.value			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 if isinstance(UpperCAmelCase__			,       (list, tuple)       ):
  return value[int(UpperCAmelCase__       )]
 if index in value:
  return value[index]
 if isinstance(UpperCAmelCase__			,       UpperCAmelCase__       ) and isinstance(UpperCAmelCase__			,       UpperCAmelCase__       ):
  a							=		difflib.get_close_matches(UpperCAmelCase__			,       list(value.keys()       )       )
  if len(UpperCAmelCase__       ) > 0:
   return value[close_matches[0]]

 raise InterpretorError(F"""Could not index {value} with '{index}'."""       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:Tuple			,       UpperCAmelCase__			:Optional[Any]			,       UpperCAmelCase__			:int       ):






 '''simple docstring'''

 if name.id in state:
  return state[name.id]
 a							=		difflib.get_close_matches(name.id			,       list(state.keys()       )       )
 if len(UpperCAmelCase__       ) > 0:
  return state[close_matches[0]]
 raise InterpretorError(F"""The variable `{name.id}` is not defined."""       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:Optional[Any]			,       UpperCAmelCase__			:Optional[int]			,       UpperCAmelCase__			:Optional[Any]       ):






 '''simple docstring'''

 if len(condition.ops       ) > 1:
  raise InterpretorError("Cannot evaluate conditions with multiple operators"       )

 a							=		evaluate_ast(condition.left			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 a							=		condition.ops[0]
 a							=		evaluate_ast(condition.comparators[0]			,       UpperCAmelCase__			,       UpperCAmelCase__       )

 if isinstance(UpperCAmelCase__			,       ast.Eq       ):
  return left == right
 elif isinstance(UpperCAmelCase__			,       ast.NotEq       ):
  return left != right
 elif isinstance(UpperCAmelCase__			,       ast.Lt       ):
  return left < right
 elif isinstance(UpperCAmelCase__			,       ast.LtE       ):
  return left <= right
 elif isinstance(UpperCAmelCase__			,       ast.Gt       ):
  return left > right
 elif isinstance(UpperCAmelCase__			,       ast.GtE       ):
  return left >= right
 elif isinstance(UpperCAmelCase__			,       ast.Is       ):
  return left is right
 elif isinstance(UpperCAmelCase__			,       ast.IsNot       ):
  return left is not right
 elif isinstance(UpperCAmelCase__			,       ast.In       ):
  return left in right
 elif isinstance(UpperCAmelCase__			,       ast.NotIn       ):
  return left not in right
 else:
  raise InterpretorError(F"""Operator not supported: {comparator}"""       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:int			,       UpperCAmelCase__			:List[str]			,       UpperCAmelCase__			:List[Any]       ):






 '''simple docstring'''

 a							=		None
 if evaluate_condition(if_statement.test			,       UpperCAmelCase__			,       UpperCAmelCase__       ):
  for line in if_statement.body:
   a							=		evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
   if line_result is not None:
    a							=		line_result
 else:
  for line in if_statement.orelse:
   a							=		evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
   if line_result is not None:
    a							=		line_result
 return result




def 						UpperCAmelCase__   (   UpperCAmelCase__			:Dict			,       UpperCAmelCase__			:List[str]			,       UpperCAmelCase__			:Union[str, Any]       ):






 '''simple docstring'''

 a							=		None
 a							=		evaluate_ast(for_loop.iter			,       UpperCAmelCase__			,       UpperCAmelCase__       )
 for counter in iterator:
  a							=		counter
  for expression in for_loop.body:
   a							=		evaluate_ast(UpperCAmelCase__			,       UpperCAmelCase__			,       UpperCAmelCase__       )
   if line_result is not None:
    a							=		line_result
 return result






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


__SCREAMING_SNAKE_CASE     =	logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     =	{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__SCREAMING_SNAKE_CASE     =	{
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json""",
    },
}

class 						lowerCamelCase_     (			_A    ):






						'''simple docstring'''







						def __init__(						self   :      Dict						,						__lowerCamelCase   :      str=None						,						__lowerCamelCase   :      int=None						,						__lowerCamelCase   :      Optional[Any]=False						,						__lowerCamelCase   :      List[Any]=True						,						__lowerCamelCase   :      int=False						,						__lowerCamelCase   :      Optional[int]="<s>"						,						__lowerCamelCase   :      Union[str, Any]="</s>"						,						__lowerCamelCase   :      Dict="<unk>"						,						__lowerCamelCase   :      List[str]="<sep>"						,						__lowerCamelCase   :      Any="<pad>"						,						__lowerCamelCase   :      Optional[Any]="<cls>"						,						__lowerCamelCase   :      Union[str, Any]="<mask>"						,						__lowerCamelCase   :      List[str]=["<eop>", "<eod>"]						,						**__lowerCamelCase   :      Optional[Any]						,						)   ->					Optional[int]:
									A				:       Any														=    AddedToken(__lowerCamelCase						,						lstrip=__lowerCamelCase						,						rstrip=__lowerCamelCase			) if isinstance(__lowerCamelCase						,						__lowerCamelCase			) else mask_token

									super().__init__(
									    vocab_file=__lowerCamelCase						,						tokenizer_file=__lowerCamelCase						,						do_lower_case=__lowerCamelCase						,						remove_space=__lowerCamelCase						,						keep_accents=__lowerCamelCase						,						bos_token=__lowerCamelCase						,						eos_token=__lowerCamelCase						,						unk_token=__lowerCamelCase						,						sep_token=__lowerCamelCase						,						pad_token=__lowerCamelCase						,						cls_token=__lowerCamelCase						,						mask_token=__lowerCamelCase						,						additional_special_tokens=__lowerCamelCase						,						**__lowerCamelCase						,						)

									A				:       str														=    3
									A				:       str														=    do_lower_case
									A				:       int														=    remove_space
									A				:       Optional[int]														=    keep_accents
									A				:       Tuple														=    vocab_file
									A				:       int														=    False if not self.vocab_file else True

									try:
												import jieba
									except ModuleNotFoundError as error:
												raise error.__class__(
												    "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
												    "See https://pypi.org/project/jieba/ for installation."			)
									A				:       str														=    jieba
									A				:       Optional[Any]														=    str.maketrans(" \n"						,						"\u2582\u2583"			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      List[str]						,						__lowerCamelCase   :      List[int]						,						__lowerCamelCase   :      Optional[List[int]] = None			)   ->					List[int]:
									A				:       Dict														=    [self.sep_token_id]
									A				:       Tuple														=    [self.cls_token_id]
									if token_ids_a is None:
												return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls







						def       SCREAMING_SNAKE_CASE__		(						self   :      int						,						__lowerCamelCase   :      List[int]						,						__lowerCamelCase   :      Optional[List[int]] = None			)   ->					List[int]:
									A				:       Tuple														=    [self.sep_token_id]
									A				:       Dict														=    [2]

									if token_ids_a is None:
												return len(token_ids_a + sep			) * [0] + cls_segment_id
									return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id







						def       SCREAMING_SNAKE_CASE__		(						self   :      int						,						__lowerCamelCase   :      str						,						__lowerCamelCase   :      Optional[str] = None			)   ->					Tuple[str]:
									if not self.can_save_slow_tokenizer:
												raise ValueError(
												    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
												    "tokenizer."			)

									if not os.path.isdir(__lowerCamelCase			):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
												return
									A				:       List[Any]														=    os.path.join(
									    __lowerCamelCase						,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)

									if os.path.abspath(self.vocab_file			) != os.path.abspath(__lowerCamelCase			):
												copyfile(self.vocab_file						,						__lowerCamelCase			)

									return (out_vocab_file,)







						def       SCREAMING_SNAKE_CASE__		(						self   :      Any						,						__lowerCamelCase   :      Dict						,						*__lowerCamelCase   :      List[str]						,						**__lowerCamelCase   :      Tuple			)   ->					str:
									A				:       Union[str, Any]														=    [
									    " ".join([x.translate(self.translator			) for x in self.jieba.cut(__lowerCamelCase						,						cut_all=__lowerCamelCase			)]			)
									    for text in batch_text_or_text_pairs
									]
									return super()._batch_encode_plus(__lowerCamelCase						,						*__lowerCamelCase						,						**__lowerCamelCase			)







						def       SCREAMING_SNAKE_CASE__		(						self   :      Any						,						*__lowerCamelCase   :      Optional[Any]						,						**__lowerCamelCase   :      Optional[int]			)   ->					Tuple:
									A				:       Optional[Any]														=    super()._decode(*__lowerCamelCase						,						**__lowerCamelCase			)
									A				:       Tuple														=    text.replace(" "						,						""			).replace("\u2582"						,						" "			).replace("\u2583"						,						"\n"			)
									return text