def hapmap3(data_set='<STR_LIT>'):
try:<EOL><INDENT>from pandas import read_pickle, DataFrame<EOL>from sys import stdout<EOL>import bz2<EOL><DEDENT>except ImportError as i:<EOL><INDENT>raise i("<STR_LIT>")<EOL><DEDENT>dir_path = os.path.join(data_path,'<STR_LIT>')<EOL>hapmap_file_name = '<STR_LIT>'<EOL>unpacked_files = [os.path.join(dir_path, hapmap_file_name+ending) for ending in ['<STR_LIT>', '<STR_LIT>']]<EOL>unpacked_files_exist = reduce(lambda a, b:a and b, map(os.path.exists, unpacked_files))<EOL>if not unpacked_files_exist and not data_available(data_set):<EOL><INDENT>download_data(data_set)<EOL><DEDENT>preprocessed_data_paths = [os.path.join(dir_path,hapmap_file_name + file_name) for file_name in['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']]<EOL>if not reduce(lambda a,b: a and b, map(os.path.exists, preprocessed_data_paths)):<EOL><INDENT>if not overide_manual_authorize and not prompt_user("<STR_LIT>"<EOL>"<STR_LIT>"):<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>status = "<STR_LIT>"<EOL>print(status)<EOL>def write_status(message, progress, status):<EOL><INDENT>stdout.write("<STR_LIT:U+0020>"*len(status)); stdout.write("<STR_LIT:\r>"); stdout.flush()<EOL>status = r"<STR_LIT>".format(message=message, ll=<NUM_LIT:20>,<EOL>perc="<STR_LIT:=>"*int(<NUM_LIT>*progress/<NUM_LIT>))<EOL>stdout.write(status); stdout.flush()<EOL>return status<EOL><DEDENT>if not unpacked_files_exist:<EOL><INDENT>status=write_status('<STR_LIT>', <NUM_LIT:0>, '<STR_LIT>')<EOL>curr = <NUM_LIT:0><EOL>for newfilepath in unpacked_files:<EOL><INDENT>if not os.path.exists(newfilepath):<EOL><INDENT>filepath = newfilepath + '<STR_LIT>'<EOL>file_size = os.path.getsize(filepath)<EOL>with open(newfilepath, '<STR_LIT:wb>') as new_file, open(filepath, '<STR_LIT:rb>') as f:<EOL><INDENT>decomp = bz2.BZ2Decompressor()<EOL>file_processed = <NUM_LIT:0><EOL>buffsize = <NUM_LIT:100> * <NUM_LIT><EOL>for data in iter(lambda : f.read(buffsize), b'<STR_LIT>'):<EOL><INDENT>new_file.write(decomp.decompress(data))<EOL>file_processed += len(data)<EOL>status=write_status('<STR_LIT>', curr+<NUM_LIT>*file_processed/(file_size), status)<EOL><DEDENT><DEDENT><DEDENT>curr += <NUM_LIT:12><EOL>status=write_status('<STR_LIT>', curr, status)<EOL>os.remove(filepath)<EOL><DEDENT><DEDENT>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snpstrnp = np.loadtxt(unpacked_files[<NUM_LIT:0>], dtype=str)<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>mapnp = np.loadtxt(unpacked_files[<NUM_LIT:1>], dtype=str)<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>infodf = DataFrame.from_csv(os.path.join(dir_path,'<STR_LIT>'), header=<NUM_LIT:0>, sep='<STR_LIT:\t>')<EOL>infodf.set_index('<STR_LIT>', inplace=<NUM_LIT:1>)<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snpstr = snpstrnp[:,<NUM_LIT:6>:].astype('<STR_LIT>').reshape(snpstrnp.shape[<NUM_LIT:0>], -<NUM_LIT:1>, <NUM_LIT:2>)<EOL>inan = snpstr[:,:,<NUM_LIT:0>] == '<STR_LIT:0>'<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>ref = np.array(map(lambda x: np.unique(x)[-<NUM_LIT:2>:], snpstr.swapaxes(<NUM_LIT:0>,<NUM_LIT:1>)[:,:,:]))<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snps = (snpstr==ref[None,:,:])<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snps = (snps*np.array([<NUM_LIT:1>,-<NUM_LIT:1>])[None,None,:])<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snps = snps.sum(-<NUM_LIT:1>)<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snps = snps.astype('<STR_LIT>')<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snps[inan] = -<NUM_LIT><EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>metaheader = np.r_[['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']]<EOL>metadf = DataFrame(columns=metaheader, data=snpstrnp[:,:<NUM_LIT:6>])<EOL>metadf.set_index('<STR_LIT>', inplace=<NUM_LIT:1>)<EOL>metadf = metadf.join(infodf.population)<EOL>metadf.to_pickle(preprocessed_data_paths[<NUM_LIT:1>])<EOL>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>snpsdf = DataFrame(index=metadf.index, data=snps, columns=mapnp[:,<NUM_LIT:1>])<EOL>with open(preprocessed_data_paths[<NUM_LIT:0>], '<STR_LIT:wb>') as f:<EOL><INDENT>pickle.dump(f, snpsdf, protocoll=-<NUM_LIT:1>)<EOL><DEDENT>status=write_status('<STR_LIT>', <NUM_LIT>, status)<EOL>inandf = DataFrame(index=metadf.index, data=inan, columns=mapnp[:,<NUM_LIT:1>])<EOL>inandf.to_pickle(preprocessed_data_paths[<NUM_LIT:2>])<EOL>status=write_status('<STR_LIT>', <NUM_LIT:100>, status)<EOL>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>snpsdf = read_pickle(preprocessed_data_paths[<NUM_LIT:0>])<EOL>print("<STR_LIT>")<EOL>metadf = read_pickle(preprocessed_data_paths[<NUM_LIT:1>])<EOL>print("<STR_LIT>")<EOL>inandf = read_pickle(preprocessed_data_paths[<NUM_LIT:2>])<EOL><DEDENT>snps = snpsdf.values<EOL>populations = metadf.population.values.astype('<STR_LIT>')<EOL>hapmap = dict(name=data_set,<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>snpsdf=snpsdf,<EOL>metadf=metadf,<EOL>snps=snps,<EOL>inan=inandf.values,<EOL>inandf=inandf,<EOL>populations=populations)<EOL>return hapmap<EOL>
The HapMap phase three SNP dataset - 1184 samples out of 11 populations.

SNP_matrix (A) encoding [see Paschou et all. 2007 (PCA-Correlated SNPs...)]:
Let (B1,B2) be the alphabetically sorted bases, which occur in the j-th SNP, then

      /  1, iff SNPij==(B1,B1)
Aij = |  0, iff SNPij==(B1,B2)
      \ -1, iff SNPij==(B2,B2)

The SNP data and the meta information (such as iid, sex and phenotype) are
stored in the dataframe datadf, index is the Individual ID,
with following columns for metainfo:

    * family_id   -> Family ID
    * paternal_id -> Paternal ID
    * maternal_id -> Maternal ID
    * sex         -> Sex (1=male; 2=female; other=unknown)
    * phenotype   -> Phenotype (-9, or 0 for unknown)
    * population  -> Population string (e.g. 'ASW' - 'YRI')
    * rest are SNP rs (ids)

More information is given in infodf:

    * Chromosome:
        - autosomal chromosemes                -> 1-22
        - X    X chromosome                    -> 23
        - Y    Y chromosome                    -> 24
        - XY   Pseudo-autosomal region of X    -> 25
        - MT   Mitochondrial                   -> 26
    * Relative Positon (to Chromosome) [base pairs]