
"""simple docstring"""




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
					import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__      :    List[Any]    							= logging.get_logger(__name__)

UpperCAmelCase__      :    Union[str, Any]    							= {'vocab_file': 'vocab.txt'}

UpperCAmelCase__      :    Dict    							= {
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

UpperCAmelCase__      :    Optional[Any]    							= {
    'openbmb/cpm-ant-10b': 1_0_2_4,
}

def lowercase_  (   _snake_case						):
							SCREAMING_SNAKE_CASE__						:       str           =   collections.OrderedDict()
							with open(_snake_case   ,"""r"""   ,encoding="""utf-8"""						) as reader:
														SCREAMING_SNAKE_CASE__						:       List[Any]           =   reader.readlines()
							for index, token in enumerate(_snake_case						):
														SCREAMING_SNAKE_CASE__						:       List[str]           =   token.rstrip("""\n"""						)
														SCREAMING_SNAKE_CASE__						:       Tuple           =   index
							return vocab



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				def __init__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__="<unk>"   ,       SCREAMING_SNAKE_CASE__=2_00    )					->		Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[Any]           =   vocab
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   unk_token
											SCREAMING_SNAKE_CASE__						:       Tuple           =   max_input_chars_per_word



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Tuple           =   list(SCREAMING_SNAKE_CASE__    )
											if len(SCREAMING_SNAKE_CASE__    ) > self.max_input_chars_per_word:
																		return [self.unk_token]

											SCREAMING_SNAKE_CASE__						:       str           =   0
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   []
											while start < len(SCREAMING_SNAKE_CASE__    ):
																		SCREAMING_SNAKE_CASE__						:       Any           =   len(SCREAMING_SNAKE_CASE__    )
																		SCREAMING_SNAKE_CASE__						:       Tuple           =   None
																		while start < end:
																									SCREAMING_SNAKE_CASE__						:       Any           =   """""".join(chars[start:end]    )
																									if substr in self.vocab:
																																SCREAMING_SNAKE_CASE__						:       List[str]           =   substr
																																break
																									end -= 1
																		if cur_substr is None:
																									sub_tokens.append(self.unk_token    )
																									start += 1
																		else:
																									sub_tokens.append(SCREAMING_SNAKE_CASE__    )
																									SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   end

											return sub_tokens






class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Any					       =   VOCAB_FILES_NAMES
				__UpperCamelCase      :						str					       =   PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase      :						Optional[Any]					       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase      :						Tuple					       =   ['''input_ids''', '''attention_mask''']
				__UpperCamelCase      :						Any					       =   False



				def __init__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__="<d>"   ,       SCREAMING_SNAKE_CASE__="</d>"   ,       SCREAMING_SNAKE_CASE__="<s>"   ,       SCREAMING_SNAKE_CASE__="</s>"   ,       SCREAMING_SNAKE_CASE__="<pad>"   ,       SCREAMING_SNAKE_CASE__="<unk>"   ,       SCREAMING_SNAKE_CASE__="</n>"   ,       SCREAMING_SNAKE_CASE__="</_>"   ,       SCREAMING_SNAKE_CASE__="left"   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		int:



											"""simple docstring"""




											requires_backends(self   ,       ["""jieba"""]    )
											super().__init__(
											    bod_token=SCREAMING_SNAKE_CASE__   ,       eod_token=SCREAMING_SNAKE_CASE__   ,       bos_token=SCREAMING_SNAKE_CASE__   ,       eos_token=SCREAMING_SNAKE_CASE__   ,       pad_token=SCREAMING_SNAKE_CASE__   ,       unk_token=SCREAMING_SNAKE_CASE__   ,       line_token=SCREAMING_SNAKE_CASE__   ,       space_token=SCREAMING_SNAKE_CASE__   ,       padding_side=SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__   ,       )
											SCREAMING_SNAKE_CASE__						:       int           =   bod_token
											SCREAMING_SNAKE_CASE__						:       Any           =   eod_token
											SCREAMING_SNAKE_CASE__						:       str           =   load_vocab(SCREAMING_SNAKE_CASE__    )
											SCREAMING_SNAKE_CASE__						:       List[str]           =   self.encoder[space_token]
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   self.encoder[line_token]

											del self.encoder[space_token]
											del self.encoder[line_token]

											SCREAMING_SNAKE_CASE__						:       Dict           =   collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda SCREAMING_SNAKE_CASE__    : x[1]    )    )
											SCREAMING_SNAKE_CASE__						:       List[str]           =   {v: k for k, v in self.encoder.items()}

											SCREAMING_SNAKE_CASE__						:       Tuple           =   WordpieceTokenizer(vocab=self.encoder   ,       unk_token=self.unk_token    )



				@property
				def 	__magic_name__    (self    )					->		Tuple:



											"""simple docstring"""




											return self.encoder[self.bod_token]



				@property
				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											return self.encoder[self.eod_token]



				@property
				def 	__magic_name__    (self    )					->		Tuple:



											"""simple docstring"""




											return self.encoder["\n"]



				@property
				def 	__magic_name__    (self    )					->		int:



											"""simple docstring"""




											return len(self.encoder    )



				def 	__magic_name__    (self    )					->		int:



											"""simple docstring"""




											return dict(self.encoder   ,       **self.added_tokens_encoder    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       int           =   []
											for x in jieba.cut(SCREAMING_SNAKE_CASE__   ,       cut_all=SCREAMING_SNAKE_CASE__    ):
																		output_tokens.extend(self.wordpiece_tokenizer.tokenize(SCREAMING_SNAKE_CASE__    )    )
											return output_tokens



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       int           =   [i for i in token_ids if i >= 0]
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   [
											    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
											]
											return super()._decode(SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		List[str]:



											"""simple docstring"""




											return token in self.encoder



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		str:



											"""simple docstring"""




											return "".join(SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Optional[int]:



											"""simple docstring"""




											return self.encoder.get(SCREAMING_SNAKE_CASE__   ,       self.encoder.get(self.unk_token    )    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Tuple:



											"""simple docstring"""




											return self.decoder.get(SCREAMING_SNAKE_CASE__   ,       self.unk_token    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		Tuple[str]:



											"""simple docstring"""




											if os.path.isdir(SCREAMING_SNAKE_CASE__    ):
																		SCREAMING_SNAKE_CASE__						:       List[str]           =   os.path.join(
																		    SCREAMING_SNAKE_CASE__   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
											else:
																		SCREAMING_SNAKE_CASE__						:       List[str]           =   (filename_prefix + """-""" if filename_prefix else """""") + save_directory
											SCREAMING_SNAKE_CASE__						:       int           =   0
											if " " in self.encoder:
																		SCREAMING_SNAKE_CASE__						:       int           =   self.encoder[""" """]
																		del self.encoder[" "]
											if "\n" in self.encoder:
																		SCREAMING_SNAKE_CASE__						:       Dict           =   self.encoder["""\n"""]
																		del self.encoder["\n"]
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda SCREAMING_SNAKE_CASE__    : x[1]    )    )
											with open(SCREAMING_SNAKE_CASE__   ,       """w"""   ,       encoding="""utf-8"""    ) as writer:
																		for token, token_index in self.encoder.items():
																									if index != token_index:
																																logger.warning(
																																    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																																    """ Please check that the vocabulary is not corrupted!"""    )
																																SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   token_index
																									writer.write(token + """\n"""    )
																									index += 1
											return (vocab_file,)



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		List[int]:



											"""simple docstring"""




											if token_ids_a is None:
																		return [self.bos_token_id] + token_ids_a
											return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None   ,       SCREAMING_SNAKE_CASE__ = False    )					->		List[int]:



											"""simple docstring"""




											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=SCREAMING_SNAKE_CASE__   ,       token_ids_a=SCREAMING_SNAKE_CASE__   ,       already_has_special_tokens=SCREAMING_SNAKE_CASE__    )

											if token_ids_a is not None:
																		return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__    ))
											return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    ))






"""simple docstring"""




import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


UpperCAmelCase__      :    Optional[int]    							= {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

UpperCAmelCase__      :    List[Any]    							= logging.WARNING

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   os.getenv("""DATASETS_VERBOSITY"""   ,_snake_case						)
							if env_level_str:
														if env_level_str in log_levels:
																					return log_levels[env_level_str]
														else:
																					logging.getLogger().warning(
																					    f'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
																					    f'''has to be one of: { ', '.join(log_levels.keys()						) }'''						)
							return _default_log_level

def lowercase_  (   ):
							return __name__.split("""."""						)[0]

def lowercase_  (   ):
							return logging.getLogger(_get_library_name()						)

def lowercase_  (   ):
							# Apply our default configuration to the library root logger.
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_root_logger()
							library_root_logger.setLevel(_get_default_logging_level()						)

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_root_logger()
							library_root_logger.setLevel(logging.NOTSET						)

def lowercase_  (   _snake_case = None						):
							if name is None:
														SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   _get_library_name()
							return logging.getLogger(_snake_case						)

def lowercase_  (   ):
							return _get_library_root_logger().getEffectiveLevel()

def lowercase_  (   _snake_case						):
							_get_library_root_logger().setLevel(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							return set_verbosity(_snake_case						)

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Tuple           =   False

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       str           =   True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()



class 	lowerCAmelCase_						:


				"""simple docstring"""



				def __init__(self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		int:  # pylint: disable=unused-argument



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[str]           =   args[0] if args else None



				def __iter__(self    )					->		int:



											"""simple docstring"""




											return iter(self._iterator    )



				def __getattr__(self   ,       SCREAMING_SNAKE_CASE__    )					->		int:



											"""simple docstring"""




											def empty_fn(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    ):  # pylint: disable=unused-argument
																		return

											return empty_fn



				def __enter__(self    )					->		Dict:



											"""simple docstring"""




											return self



				def __exit__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )					->		str:



											"""simple docstring"""




											return






UpperCAmelCase__      :    str    							= True



class 	lowerCAmelCase_						:


				"""simple docstring"""



				def __call__(self   ,       *SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__=False   ,       **SCREAMING_SNAKE_CASE__    )					->		List[Any]:



											"""simple docstring"""




											if _tqdm_active and not disable:
																		return tqdm_lib.tqdm(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )
											else:
																		return EmptyTqdm(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       *SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )					->		int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[Any]           =   None
											if _tqdm_active:
																		return tqdm_lib.tqdm.set_lock(*SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											if _tqdm_active:
																		return tqdm_lib.tqdm.get_lock()






UpperCAmelCase__      :    Tuple    							= _tqdm_cls()

def lowercase_  (   ):
							global _tqdm_active
							return bool(_tqdm_active						)

def lowercase_  (   ):
							global _tqdm_active
							SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   True

def lowercase_  (   ):
							global _tqdm_active
							SCREAMING_SNAKE_CASE__						:       str           =   False




