





#
# Sort large text files in a minimum amount of memory
#
import argparse
import os





class A_   :
   lowerCAmelCase__  	=					"""block_{0}.dat"""




   def __init__(self      :str       ,  _UpperCamelCase      :Optional[Any]					)->   Dict:
         __A			  =							filename
         __A			  =							[]




   def _lowerCAmelCase (self      :Any       ,  _UpperCamelCase      :Tuple       ,  _UpperCamelCase      :int					)->   Tuple:
         __A			  =							self.BLOCK_FILENAME_FORMAT.format(_UpperCamelCase					)
         with open(_UpperCamelCase       ,  '''w'''					) as file:
               file.write(_UpperCamelCase					)
         self.block_filenames.append(_UpperCamelCase					)




   def _lowerCAmelCase (self      :Tuple					)->   Tuple:
         return self.block_filenames




   def _lowerCAmelCase (self      :str       ,  _UpperCamelCase      :Union[str, Any]       ,  _UpperCamelCase      :List[str]=None					)->   List[str]:
         __A			  =							0
         with open(self.filename					) as file:
               while True:
                     __A			  =							file.readlines(_UpperCamelCase					)

                     if lines == []:
                           break

                     if sort_key is None:
                           lines.sort()
                     else:
                           lines.sort(key=_UpperCamelCase					)

                     self.write_block(''''''.join(_UpperCamelCase					)       ,  _UpperCamelCase					)
                     i += 1






   def _lowerCAmelCase (self      :Optional[int]					)->   List[str]:
         map(os.remove       ,  self.block_filenames					)





class A_   :




   def _lowerCAmelCase (self      :str       ,  _UpperCamelCase      :Union[str, Any]					)->   Any:
         __A			  =							-1
         __A			  =							None

         for i in range(len(_UpperCamelCase					)					):
               if min_str is None or choices[i] < min_str:
                     __A			  =							i

         return min_index





class A_   :




   def __init__(self      :List[str]       ,  _UpperCamelCase      :Tuple					)->   List[Any]:
         __A			  =							files
         __A			  =							set()
         __A			  =							len(_UpperCamelCase					)
         __A			  =							{i: None for i in range(self.num_buffers					)}




   def _lowerCAmelCase (self      :Dict					)->   Dict:
         return {
             i: self.buffers[i] for i in range(self.num_buffers					) if i not in self.empty
         }




   def _lowerCAmelCase (self      :Tuple					)->   Union[str, Any]:
         for i in range(self.num_buffers					):
               if self.buffers[i] is None and i not in self.empty:
                     __A			  =							self.files[i].readline()

                     if self.buffers[i] == "":
                           self.empty.add(_UpperCamelCase					)
                           self.files[i].close()

         if len(self.empty					) == self.num_buffers:
               return False

         return True






   def _lowerCAmelCase (self      :Any       ,  _UpperCamelCase      :List[str]					)->   Any:
         __A			  =							self.buffers[index]
         __A			  =							None

         return value





class A_   :




   def __init__(self      :int       ,  _UpperCamelCase      :Optional[int]					)->   Optional[Any]:
         __A			  =							merge_strategy




   def _lowerCAmelCase (self      :Optional[int]       ,  _UpperCamelCase      :List[str]       ,  _UpperCamelCase      :Any       ,  _UpperCamelCase      :str					)->   List[Any]:
         __A			  =							FilesArray(self.get_file_handles(_UpperCamelCase       ,  _UpperCamelCase					)					)
         with open(_UpperCamelCase       ,  '''w'''       ,  _UpperCamelCase					) as outfile:
               while buffers.refresh():
                     __A			  =							self.merge_strategy.select(buffers.get_dict()					)
                     outfile.write(buffers.unshift(_UpperCamelCase					)					)






   def _lowerCAmelCase (self      :str       ,  _UpperCamelCase      :Dict       ,  _UpperCamelCase      :List[Any]					)->   Dict:
         __A			  =							{}

         for i in range(len(_UpperCamelCase					)					):
               __A			  =							open(filenames[i]       ,  '''r'''       ,  _UpperCamelCase					)  # noqa: UP015

         return files





class A_   :




   def __init__(self      :List[str]       ,  _UpperCamelCase      :Union[str, Any]					)->   List[Any]:
         __A			  =							block_size




   def _lowerCAmelCase (self      :Union[str, Any]       ,  _UpperCamelCase      :Tuple       ,  _UpperCamelCase      :List[Any]=None					)->   str:
         __A			  =							self.get_number_blocks(_UpperCamelCase       ,  self.block_size					)
         __A			  =							FileSplitter(_UpperCamelCase					)
         splitter.split(self.block_size       ,  _UpperCamelCase					)

         __A			  =							FileMerger(NWayMerge()					)
         __A			  =							self.block_size / (num_blocks + 1)
         merger.merge(splitter.get_block_filenames()       ,  filename + '''.out'''       ,  _UpperCamelCase					)

         splitter.cleanup()






   def _lowerCAmelCase (self      :Tuple       ,  _UpperCamelCase      :Tuple       ,  _UpperCamelCase      :Union[str, Any]					)->   int:
         return (os.stat(_UpperCamelCase					).st_size / block_size) + 1





def _a							(      lowerCamelCase:  Any					)      ->		List[str]:







      '''simple docstring'''






      if string[-1].lower() == "k":
            return int(string[:-1]					) * 10_24
      elif string[-1].lower() == "m":
            return int(string[:-1]					) * 10_24 * 10_24
      elif string[-1].lower() == "g":
            return int(string[:-1]					) * 10_24 * 10_24 * 10_24
      else:
            return int(lowerCamelCase					)



def _a							(      )      ->		Dict:







      '''simple docstring'''






      __A			  =							argparse.ArgumentParser()
      parser.add_argument(
          '''-m'''   ,							'''--mem'''   ,							help='''amount of memory to use for sorting'''   ,							default='''100M'''					)
      parser.add_argument(
          '''filename'''   ,							metavar='''<filename>'''   ,							nargs=1   ,							help='''name of file to sort'''					)
      __A			  =							parser.parse_args()

      __A			  =							ExternalSort(parse_memory(args.mem					)					)
      sorter.sort(args.filename[0]					)


if __name__ == "__main__":
       main()












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


snake_case__   :				Tuple						=	{
    'configuration_realm': ['REALM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RealmConfig'],
    'tokenization_realm': ['RealmTokenizer'],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__   :				Tuple						=	['RealmTokenizerFast']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__   :				str						=	[
           'REALM_PRETRAINED_MODEL_ARCHIVE_LIST',
           'RealmEmbedder',
           'RealmForOpenQA',
           'RealmKnowledgeAugEncoder',
           'RealmPreTrainedModel',
           'RealmReader',
           'RealmScorer',
           'load_tf_weights_in_realm',
       ]
       snake_case__   :				Any						=	['RealmRetriever']


if TYPE_CHECKING:
       from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
       from .tokenization_realm import RealmTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_realm import RealmTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_realm import (
                  REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  RealmEmbedder,
                  RealmForOpenQA,
                  RealmKnowledgeAugEncoder,
                  RealmPreTrainedModel,
                  RealmReader,
                  RealmScorer,
                  load_tf_weights_in_realm,
              )
              from .retrieval_realm import RealmRetriever


else:
       import sys

       snake_case__   :				Any						=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





