






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available






def  UpperCAmelCase__						(    lowerCamelCase	:			Any			,lowerCamelCase	:			int			,lowerCamelCase	:			Optional[Any]			,lowerCamelCase	:			List[Any]=None    ):
   lowerCamelCase									=None
   # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
   if str(weight.dtype    ) == "torch.bfloat16":
      # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
      lowerCamelCase									=weight.view(torch.intaa    )
      lowerCamelCase									="""bfloat16"""
   lowerCamelCase									=weight.cpu().numpy()
   lowerCamelCase									=os.path.join(lowerCamelCase			,f'''{weight_name}.dat'''    )
   if index is not None:
      if dtype is None:
         lowerCamelCase									=str(array.dtype    )
      lowerCamelCase									={"""dtype""": dtype, """shape""": list(array.shape    )}
   if array.ndim == 0:
      lowerCamelCase									=array[None]
   lowerCamelCase									=np.memmap(lowerCamelCase			,dtype=array.dtype			,mode="""w+"""			,shape=array.shape    )
   lowerCamelCase									=array[:]
   file_array.flush()
   return index






def  UpperCAmelCase__						(    lowerCamelCase	:			Optional[Any]			,lowerCamelCase	:			int    ):
   lowerCamelCase									=tuple(weight_info["""shape"""]    )
   if shape == ():
      # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
      lowerCamelCase									=(1,)

   lowerCamelCase									=weight_info["""dtype"""]
   if dtype == "bfloat16":
      # NumPy does not support bfloat16 so this was saved as a int16
      lowerCamelCase									="""int16"""

   lowerCamelCase									=np.memmap(lowerCamelCase			,dtype=lowerCamelCase			,shape=lowerCamelCase			,mode="""r"""    )

   if len(weight_info["""shape"""]    ) == 0:
      lowerCamelCase									=weight[0]
   lowerCamelCase									=torch.tensor(lowerCamelCase    )
   if weight_info["dtype"] == "bfloat16":
      lowerCamelCase									=weight.view(torch.bfloataa    )

   return weight






def  UpperCAmelCase__						(    lowerCamelCase	:			str			,lowerCamelCase	:			List[Any]    ):
   if index is None or len(lowerCamelCase    ) == 0:
      # Nothing to save
      return

   lowerCamelCase									=os.path.join(lowerCamelCase			,"""index.json"""    )
   if os.path.isfile(lowerCamelCase    ):
      with open(lowerCamelCase			,"""r"""			,encoding="""utf-8"""    ) as f:
         lowerCamelCase									=json.load(lowerCamelCase    )
   else:
      lowerCamelCase									={}
   current_index.update(lowerCamelCase    )

   with open(lowerCamelCase			,"""w"""			,encoding="""utf-8"""    ) as f:
      json.dump(lowerCamelCase			,lowerCamelCase			,indent=2    )






def  UpperCAmelCase__						(    lowerCamelCase	:			Union[str, os.PathLike]			,lowerCamelCase	:			Dict[str, torch.Tensor]    ):
   os.makedirs(lowerCamelCase			,exist_ok=lowerCamelCase    )
   lowerCamelCase									={}
   for name, parameter in state_dict.items():
      lowerCamelCase									=offload_weight(lowerCamelCase			,lowerCamelCase			,lowerCamelCase			,index=lowerCamelCase    )

   # Update index
   save_offload_index(lowerCamelCase			,lowerCamelCase    )


class 		lowerCAmelCase_				(  UpperCamelCase ):





     def __init__(	self							, _UpperCAmelCase							, _UpperCAmelCase  )		->				int:
        lowerCamelCase									=dataset
        lowerCamelCase									=prefix





     def __getitem__(	self							, _UpperCAmelCase  )		->				Any:
        return self.dataset[F'''{self.prefix}{key}''']





     def __iter__(	self  )		->				Any:
        return iter([key for key in self.dataset if key.startswith(self.prefix  )]  )





     def __len__(	self  )		->				int:
        return len(self.dataset  )




class 		lowerCAmelCase_				(  UpperCamelCase ):





     def __init__(	self							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase=None							, )		->				str:
        if state_dict is None and save_folder is None:
           raise ValueError("""Need either a `state_dict` or a `save_folder` containing offloaded weights."""  )

        lowerCamelCase									={} if state_dict is None else state_dict
        lowerCamelCase									=save_folder
        if index is None and save_folder is not None:
           with open(os.path.join(_UpperCAmelCase							, """index.json"""  )  ) as f:
              lowerCamelCase									=json.load(_UpperCAmelCase  )
        lowerCamelCase									={} if index is None else index
        lowerCamelCase									=list(self.state_dict.keys()  )
        self.all_keys.extend([key for key in self.index if key not in self.all_keys]  )
        lowerCamelCase									=device





     def __getitem__(	self							, _UpperCAmelCase  )		->				Tuple:
        # State dict gets priority
        if key in self.state_dict:
           return self.state_dict[key]
        lowerCamelCase									=self.index[key]
        if weight_info.get("""safetensors_file"""  ) is not None:
           if not is_safetensors_available():
              raise ImportError("""These offloaded weights require the use of safetensors: `pip install safetensors`."""  )

           from safetensors import safe_open

           lowerCamelCase									="""cpu""" if self.device is None else self.device
           with safe_open(weight_info["""safetensors_file"""]							, framework="""pt"""							, device=_UpperCAmelCase  ) as f:
              lowerCamelCase									=f.get_tensor(weight_info.get("""weight_name"""							, _UpperCAmelCase  )  )

           if "dtype" in weight_info:
              return tensor.to(getattr(_UpperCAmelCase							, weight_info["""dtype"""]  )  )
           else:
              return tensor

        lowerCamelCase									=os.path.join(self.save_folder							, F'''{key}.dat'''  )
        return load_offloaded_weight(_UpperCAmelCase							, _UpperCAmelCase  )





     def __iter__(	self  )		->				Optional[int]:
        return iter(self.all_keys  )





     def __len__(	self  )		->				Any:
        return len(self.all_keys  )






def  UpperCAmelCase__						(    lowerCamelCase	:			Dict[str, torch.Tensor]			,lowerCamelCase	:			List[str]    ):
   lowerCamelCase									={}
   for module_name in submodule_names:
      # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
      # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
      result.update(
          {
              key: param
              for key, param in state_dict.items()
              if key == module_name or key.startswith(module_name + """."""    )
          }    )
   return result







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 		lowerCAmelCase_				(  UpperCamelCase				,     unittest.TestCase ):
     __lowerCamelCase       :    List[str] 	=     HerbertTokenizer
     __lowerCamelCase       :    Union[str, Any] 	=     HerbertTokenizerFast
     __lowerCamelCase       :    Union[str, Any] 	=     True





     def 						lowerCamelCase							(	self  )		->				int:
        super().setUp()

        # Use a simpler test file without japanese/chinese characters
        with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''							, encoding="""utf-8"""  ) as f_data:
           lowerCamelCase									=f_data.read().replace("""\n\n"""							, """\n"""  ).strip()

        lowerCamelCase									=[
            """<s>""",
            """</s>""",
            """l""",
            """o""",
            """w""",
            """e""",
            """r""",
            """s""",
            """t""",
            """i""",
            """d""",
            """n""",
            """w</w>""",
            """r</w>""",
            """t</w>""",
            """lo""",
            """low""",
            """er</w>""",
            """low</w>""",
            """lowest</w>""",
            """newer</w>""",
            """wider</w>""",
            """,</w>""",
            """<unk>""",
        ]
        lowerCamelCase									=dict(zip(_UpperCAmelCase							, range(len(_UpperCAmelCase  )  )  )  )
        lowerCamelCase									=["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

        lowerCamelCase									=os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["""vocab_file"""]  )
        lowerCamelCase									=os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["""merges_file"""]  )
        with open(self.vocab_file							, """w"""  ) as fp:
           fp.write(json.dumps(_UpperCAmelCase  )  )
        with open(self.merges_file							, """w"""  ) as fp:
           fp.write("""\n""".join(_UpperCAmelCase  )  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:
        lowerCamelCase									="""lower newer"""
        lowerCamelCase									="""lower newer"""
        return input_text, output_text





     def 						lowerCamelCase							(	self  )		->				Any:
        lowerCamelCase									=self.tokenizer_class(vocab_file=self.vocab_file							, merges_file=self.merges_file  )

        lowerCamelCase									="""lower"""
        lowerCamelCase									=["""low""", """er</w>"""]
        lowerCamelCase									=tokenizer.tokenize(_UpperCAmelCase  )
        self.assertListEqual(_UpperCAmelCase							, _UpperCAmelCase  )

        lowerCamelCase									=tokens + ["""<unk>"""]
        lowerCamelCase									=[16, 17, 23]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase  )							, _UpperCAmelCase  )





     def 						lowerCamelCase							(	self  )		->				Union[str, Any]:
        if not self.test_rust_tokenizer:
           return

        lowerCamelCase									=self.get_tokenizer()
        lowerCamelCase									=self.get_rust_tokenizer()

        lowerCamelCase									="""lower,newer"""

        lowerCamelCase									=tokenizer.tokenize(_UpperCAmelCase  )
        lowerCamelCase									=rust_tokenizer.tokenize(_UpperCAmelCase  )
        self.assertListEqual(_UpperCAmelCase							, _UpperCAmelCase  )

        lowerCamelCase									=tokenizer.encode(_UpperCAmelCase							, add_special_tokens=_UpperCAmelCase  )
        lowerCamelCase									=rust_tokenizer.encode(_UpperCAmelCase							, add_special_tokens=_UpperCAmelCase  )
        self.assertListEqual(_UpperCAmelCase							, _UpperCAmelCase  )

        lowerCamelCase									=self.get_rust_tokenizer()
        lowerCamelCase									=tokenizer.encode(_UpperCAmelCase  )
        lowerCamelCase									=rust_tokenizer.encode(_UpperCAmelCase  )
        self.assertListEqual(_UpperCAmelCase							, _UpperCAmelCase  )





     @slow
     def 						lowerCamelCase							(	self  )		->				Any:
        lowerCamelCase									=self.tokenizer_class.from_pretrained("""allegro/herbert-base-cased"""  )

        lowerCamelCase									=tokenizer.encode("""konstruowanie sekwencji"""							, add_special_tokens=_UpperCAmelCase  )
        lowerCamelCase									=tokenizer.encode("""konstruowanie wielu sekwencji"""							, add_special_tokens=_UpperCAmelCase  )

        lowerCamelCase									=tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase  )
        lowerCamelCase									=tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase							, _UpperCAmelCase  )

        assert encoded_sentence == [0] + text + [2]
        assert encoded_pair == [0] + text + [2] + text_a + [2]