def get_genes(<EOL>path_or_buffer, valid_biotypes,<EOL>chunksize=<NUM_LIT>,<EOL>chromosome_pattern=None,<EOL>only_manual=False,<EOL>remove_duplicates=True,<EOL>sort_by='<STR_LIT:name>'):
chrompat = None<EOL>if chromosome_pattern is not None:<EOL><INDENT>chrompat = re.compile(chromosome_pattern)<EOL><DEDENT>valid_biotypes = set(valid_biotypes)<EOL>c = <NUM_LIT:0><EOL>num_lines = <NUM_LIT:0><EOL>num_chunks = <NUM_LIT:0><EOL>t0 = time.time()<EOL>reader = pd.read_csv(path_or_buffer, encoding='<STR_LIT:ascii>', sep='<STR_LIT:\t>',<EOL>header=None, comment='<STR_LIT:#>', dtype={<NUM_LIT:0>: str},<EOL>chunksize=chunksize)<EOL>valid_sources = set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>if not only_manual:<EOL><INDENT>valid_sources.add('<STR_LIT>')<EOL><DEDENT>excluded_chromosomes = set()<EOL>data = []<EOL>for j, df in enumerate(reader):<EOL><INDENT>num_chunks += <NUM_LIT:1><EOL>num_lines += (df.shape[<NUM_LIT:0>])<EOL>sel = (df.iloc[:, <NUM_LIT:2>] == '<STR_LIT>')<EOL>for i, row in df.loc[sel].iterrows():<EOL><INDENT>attr = gtf.parse_attributes(row[<NUM_LIT:8>].lstrip('<STR_LIT:U+0020>'))<EOL>biotype = attr['<STR_LIT>']<EOL>if biotype not in valid_biotypes:<EOL><INDENT>continue<EOL><DEDENT>chrom = str(row[<NUM_LIT:0>])<EOL>source = row[<NUM_LIT:1>]<EOL>if chrompat is not None:<EOL><INDENT>match = chrompat.match(chrom)<EOL>if match is None:<EOL><INDENT>excluded_chromosomes.add(chrom)<EOL>continue<EOL><DEDENT><DEDENT>c += <NUM_LIT:1><EOL>ensembl_id = attr['<STR_LIT>']<EOL>try:<EOL><INDENT>gene_name = attr['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>gene_name = ensembl_id<EOL><DEDENT>if row[<NUM_LIT:6>] == '<STR_LIT:+>':<EOL><INDENT>pos = int(row[<NUM_LIT:3>])-<NUM_LIT:1><EOL><DEDENT>elif row[<NUM_LIT:6>] == '<STR_LIT:->':<EOL><INDENT>pos = -(int(row[<NUM_LIT:4>])-<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>% str(row[<NUM_LIT:6>]))<EOL><DEDENT>length = abs(int(row[<NUM_LIT:4>]) - int(row[<NUM_LIT:3>])) + <NUM_LIT:1><EOL>data.append([ensembl_id, gene_name, chrom, pos, length,<EOL>biotype, source])<EOL><DEDENT><DEDENT>t1 = time.time()<EOL>header = ['<STR_LIT>', '<STR_LIT:name>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT:type>', '<STR_LIT:source>']<EOL>df = pd.DataFrame(columns=header, data=data)<EOL>if '<STR_LIT>' in valid_biotypes:<EOL><INDENT>if only_manual:<EOL><INDENT>sel = (df['<STR_LIT:type>'] == '<STR_LIT>' & df['<STR_LIT:source>'] == '<STR_LIT>')<EOL>df = df.loc[~sel]<EOL><DEDENT>else:<EOL><INDENT>sel_pc = df['<STR_LIT:type>'] == '<STR_LIT>'<EOL>sel_ensembl = ((df['<STR_LIT:source>'] == '<STR_LIT>') & sel_pc)<EOL>sel_manual = ((df['<STR_LIT:source>'] != '<STR_LIT>') & sel_pc)<EOL>redundant_ensembl_genes = set(df.loc[sel_ensembl, '<STR_LIT:name>'].values)& set(df.loc[sel_manual, '<STR_LIT:name>'].values)<EOL>sel_redund = sel_ensembl & df['<STR_LIT:name>'].isin(redundant_ensembl_genes)<EOL>num_genes_before = df.shape[<NUM_LIT:0>]<EOL>df = df.loc[~sel_redund]<EOL>num_genes_after = df.shape[<NUM_LIT:0>]<EOL>_LOGGER.info('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>num_genes_before - num_genes_after)<EOL><DEDENT>if remove_duplicates:<EOL><INDENT>num_genes_before = df.shape[<NUM_LIT:0>]<EOL>sel_pc = df['<STR_LIT:type>'] == '<STR_LIT>'<EOL>df_sel = df.loc[sel_pc].copy()<EOL>df_sel.sort_values('<STR_LIT>', kind='<STR_LIT>', inplace=True)<EOL>df_sel.drop_duplicates('<STR_LIT:name>', inplace=True)<EOL>df = pd.concat([df_sel, df.loc[~sel_pc]])<EOL>df.sort_index(inplace=True)<EOL>num_genes_after = df.shape[<NUM_LIT:0>]<EOL>_LOGGER.info('<STR_LIT>',<EOL>num_genes_before - num_genes_after)<EOL><DEDENT>else:<EOL><INDENT>sel = df['<STR_LIT:type>'] == '<STR_LIT>'<EOL>counts = df.loc[sel]['<STR_LIT:name>'].value_counts()<EOL>sel = counts > <NUM_LIT:1><EOL>if sel.sum() > <NUM_LIT:0>:<EOL><INDENT>_LOGGER.info('<STR_LIT>'<EOL>'<STR_LIT:%s>', '<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>' % (k, v)<EOL>for k, v in counts[sel].items()]))<EOL><DEDENT><DEDENT><DEDENT>if sort_by == '<STR_LIT:name>':<EOL><INDENT>df.sort_values(['<STR_LIT:name>'], kind='<STR_LIT>', inplace=True)<EOL><DEDENT>elif sort_by in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>df_sort = pd.concat([df['<STR_LIT>'], df['<STR_LIT>'].abs()], axis=<NUM_LIT:1>)<EOL>df_sort = df_sort.sort_values(['<STR_LIT>', '<STR_LIT>'],<EOL>kind='<STR_LIT>')<EOL>df = df.loc[df_sort.index]<EOL>if sort_by == '<STR_LIT>':<EOL><INDENT>def transform_chrom(chrom):<EOL><INDENT>"""<STR_LIT>"""<EOL>try:<EOL><INDENT>c = int(chrom)<EOL><DEDENT>except:<EOL><INDENT>if chrom in ['<STR_LIT:X>', '<STR_LIT:Y>']:<EOL><INDENT>return chrom<EOL><DEDENT>elif chrom == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'  <EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' + chrom  <EOL><DEDENT><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % c<EOL><DEDENT><DEDENT>chrom_for_sorting = df['<STR_LIT>'].apply(transform_chrom)<EOL>a = chrom_for_sorting.argsort(kind='<STR_LIT>')<EOL>df = df.iloc[a]<EOL>_LOGGER.info('<STR_LIT>')<EOL><DEDENT><DEDENT>df.set_index('<STR_LIT>', inplace=True)<EOL>_LOGGER.info('<STR_LIT>', num_lines, num_chunks)<EOL>_LOGGER.info('<STR_LIT>', c)<EOL>_LOGGER.info('<STR_LIT>', df.shape[<NUM_LIT:0>])<EOL>_LOGGER.info('<STR_LIT>', t1-t0)<EOL>all_chromosomes = list(df['<STR_LIT>'].unique())<EOL>_LOGGER.info('<STR_LIT>',<EOL>len(all_chromosomes),<EOL>'<STR_LIT:U+002CU+0020>'.join(all_chromosomes))<EOL>_LOGGER.info('<STR_LIT>',<EOL>len(excluded_chromosomes),<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(excluded_chromosomes)))<EOL>_LOGGER.info('<STR_LIT>')<EOL>for i, c in df['<STR_LIT:source>'].value_counts().iteritems():<EOL><INDENT>_LOGGER.info('<STR_LIT>', i, c)<EOL><DEDENT>_LOGGER.info('<STR_LIT>')<EOL>for i, c in df['<STR_LIT:type>'].value_counts().iteritems():<EOL><INDENT>_LOGGER.info('<STR_LIT>', i, c)<EOL><DEDENT>return df<EOL>
Get all genes of a specific a biotype from an Ensembl GTF file.

    Parameters
    ----------
    path_or_buffer : str or buffer
        The GTF file (either the file path or a buffer).
    valid_biotypes : set of str
        The set of biotypes to include (e.g., "protein_coding").
    chromosome_pattern : str, optional
        Regular expression specifying valid chromosomes. [None]
    only_manual : bool, optional
        Whether to exclude annotations with source "ensembl", which
        are based only on an automatic annotation pipeline. [True]
    remove_duplicates : bool, optional
        Whether to remove duplicate annotations, i.e. those with different
        Ensembl IDs for the same gene (only applies to protein-coding genes).
        [True]
    sort_by : str, optional
        How to sort the genes. One of:
          - 'name': Genes are ordered alphabetically by their name
          - 'position': Genes are sorted by their position in the genome.abs
                        Genes are first sorted by chromosome, then by their
                        starting base pair position on the chromosome.
          - 'position_fancy': Like 'position', but attempts to sort the
                              chromosomes in a more logical order than strictly
                              alphabetically. This currently works for human
                              and mouse genomes.
          - 'none': The order from the GTF file is retained. 
        Default: 'name'  

    Returns
    -------
    `pandas.DataFrame`
        Table with rows corresponding to the genes found.

    Notes
    -----

    Annotation sources and redundant gene annotations
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    According to the Ensembl website (1), the Ensembl gene annotation
    GTF files for human, mouse, zebrafish, rat and pig essentially
    contain two sets of annotations:

    One set consists of all annotations with the "ensembl"
    source annotation (column 2). These annotations are the product of
    the automated Ensembl "genebuild" pipeline.

    The other set consists of genes that are manually annotated by
    the HAVANA team (source "havana"), some of which have been merged with the
    automatic annotations (source "ensembl_havana").

    There seems to be no overlap between genes annotated with "havana" and
    "ensembl_havana" sources, respectively. However, there are a few genes for
    which only annotations with source "ensembl" exist.

    Our policy is therefore to prefer annotations with source "ensembl_havana"
    and "havana" over those with source "ensembl", and to only keep annotations
    with source "ensembl" if there are no manually curated alternative
    annotations.

    A special case is represented by mitochondrial genes, which always have the
    source "insdc".

    (1) see http://www.ensembl.org/Help/Faq?id=152


    Removal of duplicates
    ~~~~~~~~~~~~~~~~~~~~~

    Unfortunately, the Ensembl gene annotations contain duplicates for a
    handful of genes. For example, for MATR3, there are ENSG00000015479 and
    ENSG00000280987, both of type
    "ensembl_havana". There seems to be no clear criterion by which we could
    rationally and automatically choose one ID over the other, at least based
    on information contained
    in the GTF file.

    We therefore remove duplicates according to following policy:
    - For genes on '+' strand, keep the gene with the left-most starting
      position.
    - For genes on '-' strand, keep the gene with the right-most starting
      position.
    (In case the starting positions are equal, we keep the one that occurs
    first in the GTF file.)

    We would like to use the pandas.DataFrame.drop_duplicates() function for
    this. So we're temporarily reordering genes using their signed position,
    and then we're using the original index (position) to restore the original
    order.