
'''simple docstring'''





#
# Sort large text files in a minimum amount of memory
#
import argparse
import os



class 			lowerCAmelCase__ :



  '''simple docstring'''






  _UpperCamelCase							=			'block_{0}.dat'

  def __init__(self	:			List[Any]						,			lowerCAmelCase_	:			List[str]     ) ->							List[Any]:
        snake_case_    							=				filename
        snake_case_    							=				[]

  def 		_lowerCamelCase						(self	:			List[str]						,			lowerCAmelCase_	:			Union[str, Any]						,			lowerCAmelCase_	:			Union[str, Any]     ) ->							Optional[Any]:
        snake_case_    							=				self.BLOCK_FILENAME_FORMAT.format(lowerCAmelCase_     )
        with open(lowerCAmelCase_						,			"w"     ) as file:
              file.write(lowerCAmelCase_     )
        self.block_filenames.append(lowerCAmelCase_     )

  def 		_lowerCamelCase						(self	:			Any     ) ->							Optional[Any]:
        return self.block_filenames

  def 		_lowerCamelCase						(self	:			Dict						,			lowerCAmelCase_	:			Tuple						,			lowerCAmelCase_	:			Optional[Any]=None     ) ->							Union[str, Any]:
        snake_case_    							=				0
        with open(self.filename     ) as file:
              while True:
                    snake_case_    							=				file.readlines(lowerCAmelCase_     )

                    if lines == []:
                          break

                    if sort_key is None:
                          lines.sort()
                    else:
                          lines.sort(key=lowerCAmelCase_     )

                    self.write_block("".join(lowerCAmelCase_     )						,			lowerCAmelCase_     )
                    i += 1


  def 		_lowerCamelCase						(self	:			Dict     ) ->							List[str]:
        map(os.remove						,			self.block_filenames     )



class 			lowerCAmelCase__ :



  '''simple docstring'''






  def 		_lowerCamelCase						(self	:			List[str]						,			lowerCAmelCase_	:			List[Any]     ) ->							Tuple:
        snake_case_    							=				-1
        snake_case_    							=				None

        for i in range(len(lowerCAmelCase_     )     ):
              if min_str is None or choices[i] < min_str:
                    snake_case_    							=				i

        return min_index



class 			lowerCAmelCase__ :



  '''simple docstring'''






  def __init__(self	:			Optional[Any]						,			lowerCAmelCase_	:			List[str]     ) ->							List[Any]:
        snake_case_    							=				files
        snake_case_    							=				set()
        snake_case_    							=				len(lowerCAmelCase_     )
        snake_case_    							=				{i: None for i in range(self.num_buffers     )}

  def 		_lowerCamelCase						(self	:			Any     ) ->							Union[str, Any]:
        return {
            i: self.buffers[i] for i in range(self.num_buffers     ) if i not in self.empty
        }

  def 		_lowerCamelCase						(self	:			List[str]     ) ->							List[str]:
        for i in range(self.num_buffers     ):
              if self.buffers[i] is None and i not in self.empty:
                    snake_case_    							=				self.files[i].readline()

                    if self.buffers[i] == "":
                          self.empty.add(lowerCAmelCase_     )
                          self.files[i].close()

        if len(self.empty     ) == self.num_buffers:
              return False

        return True


  def 		_lowerCamelCase						(self	:			Union[str, Any]						,			lowerCAmelCase_	:			Any     ) ->							int:
        snake_case_    							=				self.buffers[index]
        snake_case_    							=				None

        return value



class 			lowerCAmelCase__ :



  '''simple docstring'''






  def __init__(self	:			Optional[int]						,			lowerCAmelCase_	:			str     ) ->							Any:
        snake_case_    							=				merge_strategy

  def 		_lowerCamelCase						(self	:			Any						,			lowerCAmelCase_	:			str						,			lowerCAmelCase_	:			List[str]						,			lowerCAmelCase_	:			List[Any]     ) ->							int:
        snake_case_    							=				FilesArray(self.get_file_handles(lowerCAmelCase_						,			lowerCAmelCase_     )     )
        with open(lowerCAmelCase_						,			"w"						,			lowerCAmelCase_     ) as outfile:
              while buffers.refresh():
                    snake_case_    							=				self.merge_strategy.select(buffers.get_dict()     )
                    outfile.write(buffers.unshift(lowerCAmelCase_     )     )


  def 		_lowerCamelCase						(self	:			str						,			lowerCAmelCase_	:			Any						,			lowerCAmelCase_	:			Optional[Any]     ) ->							Dict:
        snake_case_    							=				{}

        for i in range(len(lowerCAmelCase_     )     ):
              snake_case_    							=				open(filenames[i]						,			"r"						,			lowerCAmelCase_     )  # noqa: UP015

        return files



class 			lowerCAmelCase__ :



  '''simple docstring'''






  def __init__(self	:			int						,			lowerCAmelCase_	:			List[str]     ) ->							List[Any]:
        snake_case_    							=				block_size

  def 		_lowerCamelCase						(self	:			Any						,			lowerCAmelCase_	:			List[Any]						,			lowerCAmelCase_	:			Tuple=None     ) ->							List[str]:
        snake_case_    							=				self.get_number_blocks(lowerCAmelCase_						,			self.block_size     )
        snake_case_    							=				FileSplitter(lowerCAmelCase_     )
        splitter.split(self.block_size						,			lowerCAmelCase_     )

        snake_case_    							=				FileMerger(NWayMerge()     )
        snake_case_    							=				self.block_size / (num_blocks + 1)
        merger.merge(splitter.get_block_filenames()						,			filename + ".out"						,			lowerCAmelCase_     )

        splitter.cleanup()


  def 		_lowerCamelCase						(self	:			Optional[int]						,			lowerCAmelCase_	:			Optional[Any]						,			lowerCAmelCase_	:			Tuple     ) ->							Dict:
        return (os.stat(lowerCAmelCase_     ).st_size / block_size) + 1

def 	snake_case_							(				_a  :				Optional[int]      ):
      if string[-1].lower() == "k":
            return int(string[:-1]      ) * 1_024
      elif string[-1].lower() == "m":
            return int(string[:-1]      ) * 1_024 * 1_024
      elif string[-1].lower() == "g":
            return int(string[:-1]      ) * 1_024 * 1_024 * 1_024
      else:
            return int(_a      )
def 	snake_case_							(				):
      snake_case_    							=				argparse.ArgumentParser()
      parser.add_argument(
          "-m"			,       "--mem"			,       help="amount of memory to use for sorting"			,       default="100M"      )
      parser.add_argument(
          "filename"			,       metavar="<filename>"			,       nargs=1			,       help="name of file to sort"      )
      snake_case_    							=				parser.parse_args()

      snake_case_    							=				ExternalSort(parse_memory(args.mem      )      )
      sorter.sort(args.filename[0]      )


if __name__ == "__main__":
     main()




"""simple docstring"""



import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
 from .tokenization_deberta_va import DebertaVaTokenizer
else:
 SCREAMING_SNAKE_CASE_       :						List[str]					=    None

SCREAMING_SNAKE_CASE_       :						int					=    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_       :						Any					=    {'vocab_file': 'spm.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_       :						Optional[int]					=    {
    'vocab_file': {
        'microsoft/deberta-v2-xlarge': 'https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xxlarge': 'https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'
        ),
        'microsoft/deberta-v2-xxlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'
        ),
    }
}

SCREAMING_SNAKE_CASE_       :						Optional[Any]					=    {
    'microsoft/deberta-v2-xlarge': 5_1_2,
    'microsoft/deberta-v2-xxlarge': 5_1_2,
    'microsoft/deberta-v2-xlarge-mnli': 5_1_2,
    'microsoft/deberta-v2-xxlarge-mnli': 5_1_2,
}

SCREAMING_SNAKE_CASE_       :						Optional[Any]					=    {
    'microsoft/deberta-v2-xlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xlarge-mnli': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge-mnli': {'do_lower_case': False},
}



class        a    (							_lowerCamelCase					):



    """simple docstring"""



    UpperCAmelCase    			=       VOCAB_FILES_NAMES
    UpperCAmelCase    			=       PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase    			=       PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase    			=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase    			=       DebertaVaTokenizer

    def __init__(  self:			Optional[int]     ,    UpperCamelCase:			str=None     ,    UpperCamelCase:			str=None     ,    UpperCamelCase:			Optional[int]=False     ,    UpperCamelCase:			Optional[int]=False     ,    UpperCamelCase:			List[str]="[CLS]"     ,    UpperCamelCase:			Optional[Any]="[SEP]"     ,    UpperCamelCase:			int="[UNK]"     ,    UpperCamelCase:			List[Any]="[SEP]"     ,    UpperCamelCase:			Optional[Any]="[PAD]"     ,    UpperCamelCase:			Any="[CLS]"     ,    UpperCamelCase:			List[Any]="[MASK]"     ,    **UpperCamelCase:			List[str]     ,    ):






         """simple docstring"""


         super().__init__(
             UpperCamelCase     ,    tokenizer_file=UpperCamelCase     ,    do_lower_case=UpperCamelCase     ,    bos_token=UpperCamelCase     ,    eos_token=UpperCamelCase     ,    unk_token=UpperCamelCase     ,    sep_token=UpperCamelCase     ,    pad_token=UpperCamelCase     ,    cls_token=UpperCamelCase     ,    mask_token=UpperCamelCase     ,    split_by_punct=UpperCamelCase     ,    **UpperCamelCase     ,    )

         A__				      =	do_lower_case
         A__				      =	split_by_punct
         A__				      =	vocab_file
         A__				      =	False if not self.vocab_file else True

    def 					UpperCamelCase						(  self:			Dict     ,    UpperCamelCase:			List[Any]     ,    UpperCamelCase:			Optional[int]=None							):






         """simple docstring"""


         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         A__				      =	[self.cls_token_id]
         A__				      =	[self.sep_token_id]
         return cls + token_ids_a + sep + token_ids_a + sep

    def 					UpperCamelCase						(  self:			Tuple     ,    UpperCamelCase:			Any     ,    UpperCamelCase:			Optional[Any]=None     ,    UpperCamelCase:			Tuple=False							):






         """simple docstring"""


         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=UpperCamelCase     ,    token_ids_a=UpperCamelCase     ,    already_has_special_tokens=UpperCamelCase							)

         if token_ids_a is not None:
              return [1] + ([0] * len(UpperCamelCase							)) + [1] + ([0] * len(UpperCamelCase							)) + [1]
         return [1] + ([0] * len(UpperCamelCase							)) + [1]

    def 					UpperCamelCase						(  self:			int     ,    UpperCamelCase:			Tuple     ,    UpperCamelCase:			Dict=None							):






         """simple docstring"""


         A__				      =	[self.sep_token_id]
         A__				      =	[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





    def 					UpperCamelCase						(  self:			Dict     ,    UpperCamelCase:			str     ,    UpperCamelCase:			Optional[str] = None							):






         """simple docstring"""


         if not self.can_save_slow_tokenizer:
              raise ValueError(
                  """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                  """tokenizer."""							)

         if not os.path.isdir(UpperCamelCase							):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
              return
         A__				      =	os.path.join(
             UpperCamelCase     ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase							):
              copyfile(self.vocab_file     ,    UpperCamelCase							)

         return (out_vocab_file,)






