Source code for vcf.parser

import codecs
import collections
import csv
import gzip
import itertools
import os
import re
import sys

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

try:
    import pysam
except ImportError:
    pysam = None

try:
    import cparse
except ImportError:
    cparse = None

from model import _Call, _Record, make_calldata_tuple
from model import _Substitution, _Breakend, _SingleBreakend, _SV


# Metadata parsers/constants
RESERVED_INFO = {
    'AA': 'String', 'AC': 'Integer', 'AF': 'Float', 'AN': 'Integer',
    'BQ': 'Float', 'CIGAR': 'String', 'DB': 'Flag', 'DP': 'Integer',
    'END': 'Integer', 'H2': 'Flag', 'H3': 'Flag', 'MQ': 'Float',
    'MQ0': 'Integer', 'NS': 'Integer', 'SB': 'String', 'SOMATIC': 'Flag',
    'VALIDATED': 'Flag', '1000G': 'Flag',

    # Keys used for structural variants
    'IMPRECISE': 'Flag', 'NOVEL': 'Flag', 'SVTYPE': 'String',
    'SVLEN': 'Integer', 'CIPOS': 'Integer', 'CIEND': 'Integer',
    'HOMLEN': 'Integer', 'HOMSEQ': 'String', 'BKPTID': 'String',
    'MEINFO': 'String', 'METRANS': 'String', 'DGVID': 'String',
    'DBVARID': 'String', 'DBRIPID': 'String', 'MATEID': 'String',
    'PARID': 'String', 'EVENT': 'String', 'CILEN': 'Integer',
    'DPADJ': 'Integer', 'CN': 'Integer', 'CNADJ': 'Integer',
    'CICN': 'Integer', 'CICNADJ': 'Integer'
}

RESERVED_FORMAT = {
    'GT': 'String', 'DP': 'Integer', 'FT': 'String', 'GL': 'Float',
    'GLE': 'String', 'PL': 'Integer', 'GP': 'Float', 'GQ': 'Integer',
    'HQ': 'Integer', 'PS': 'Integer', 'PQ': 'Integer', 'EC': 'Integer',
    'MQ': 'Integer',

    # Keys used for structural variants
    'CN': 'Integer', 'CNQ': 'Float', 'CNL': 'Float', 'NQ': 'Integer',
    'HAP': 'Integer', 'AHAP': 'Integer'
}

# Spec is a bit weak on which metadata lines are singular, like fileformat
# and which can have repeats, like contig
SINGULAR_METADATA = ['fileformat', 'fileDate', 'reference']

# Conversion between value in file and Python value
field_counts = {
    '.': None,  # Unknown number of values
    'A': -1,  # Equal to the number of alternate alleles in a given record
    'G': -2,  # Equal to the number of genotypes in a given record
    'R': -3,  # Equal to the number of alleles including reference in a given record
}


_Info = collections.namedtuple('Info', ['id', 'num', 'type', 'desc', 'source', 'version'])
_Filter = collections.namedtuple('Filter', ['id', 'desc'])
_Alt = collections.namedtuple('Alt', ['id', 'desc'])
_Format = collections.namedtuple('Format', ['id', 'num', 'type', 'desc'])
_SampleInfo = collections.namedtuple('SampleInfo', ['samples', 'gt_bases', 'gt_types', 'gt_phases'])
_Contig = collections.namedtuple('Contig', ['id', 'length'])


class _vcf_metadata_parser(object):
    '''Parse the metadata in the header of a VCF file.'''
    def __init__(self):
        super(_vcf_metadata_parser, self).__init__()
        self.info_pattern = re.compile(r'''\#\#INFO=<
            ID=(?P<id>[^,]+),\s*
            Number=(?P<number>-?\d+|\.|[AGR])?,\s*
            Type=(?P<type>Integer|Float|Flag|Character|String),\s*
            Description="(?P<desc>[^"]*)"
            (?:,\s*Source="(?P<source>[^"]*)")?
            (?:,\s*Version="?(?P<version>[^"]*)"?)?
            >''', re.VERBOSE)
        self.filter_pattern = re.compile(r'''\#\#FILTER=<
            ID=(?P<id>[^,]+),\s*
            Description="(?P<desc>[^"]*)"
            >''', re.VERBOSE)
        self.alt_pattern = re.compile(r'''\#\#ALT=<
            ID=(?P<id>[^,]+),\s*
            Description="(?P<desc>[^"]*)"
            >''', re.VERBOSE)
        self.format_pattern = re.compile(r'''\#\#FORMAT=<
            ID=(?P<id>.+),\s*
            Number=(?P<number>-?\d+|\.|[AGR]),\s*
            Type=(?P<type>.+),\s*
            Description="(?P<desc>.*)"
            >''', re.VERBOSE)
        self.contig_pattern = re.compile(r'''\#\#contig=<
            ID=(?P<id>[^>,]+)
            (,.*length=(?P<length>-?\d+))?
            .*
            >''', re.VERBOSE)
        self.meta_pattern = re.compile(r'''##(?P<key>.+?)=(?P<val>.+)''')

    def vcf_field_count(self, num_str):
        """Cast vcf header numbers to integer or None"""
        if num_str is None:
            return None
        elif num_str not in field_counts:
            # Fixed, specified number
            return int(num_str)
        else:
            return field_counts[num_str]

    def read_info(self, info_string):
        '''Read a meta-information INFO line.'''
        match = self.info_pattern.match(info_string)
        if not match:
            raise SyntaxError(
                "One of the INFO lines is malformed: %s" % info_string)

        num = self.vcf_field_count(match.group('number'))

        info = _Info(match.group('id'), num,
                     match.group('type'), match.group('desc'),
                     match.group('source'), match.group('version'))

        return (match.group('id'), info)

    def read_filter(self, filter_string):
        '''Read a meta-information FILTER line.'''
        match = self.filter_pattern.match(filter_string)
        if not match:
            raise SyntaxError(
                "One of the FILTER lines is malformed: %s" % filter_string)

        filt = _Filter(match.group('id'), match.group('desc'))

        return (match.group('id'), filt)

    def read_alt(self, alt_string):
        '''Read a meta-information ALTline.'''
        match = self.alt_pattern.match(alt_string)
        if not match:
            raise SyntaxError(
                "One of the ALT lines is malformed: %s" % alt_string)

        alt = _Alt(match.group('id'), match.group('desc'))

        return (match.group('id'), alt)

    def read_format(self, format_string):
        '''Read a meta-information FORMAT line.'''
        match = self.format_pattern.match(format_string)
        if not match:
            raise SyntaxError(
                "One of the FORMAT lines is malformed: %s" % format_string)

        num = self.vcf_field_count(match.group('number'))

        form = _Format(match.group('id'), num,
                       match.group('type'), match.group('desc'))

        return (match.group('id'), form)

    def read_contig(self, contig_string):
        '''Read a meta-contigrmation INFO line.'''
        match = self.contig_pattern.match(contig_string)
        if not match:
            raise SyntaxError(
                "One of the contig lines is malformed: %s" % contig_string)
        length = self.vcf_field_count(match.group('length'))
        contig = _Contig(match.group('id'), length)
        return (match.group('id'), contig)

    def read_meta_hash(self, meta_string):
        # assert re.match("##.+=<", meta_string)
        items = meta_string.split('=', 1)
        # Removing initial hash marks
        key = items[0].lstrip('#')
        # N.B., items can have quoted values, so cannot just split on comma
        val = OrderedDict()
        state = 0
        k = ''
        v = ''
        for c in items[1].strip('[<>]'):

            if state == 0:  # reading item key
                if c == '=':
                    state = 1  # end of key, start reading value
                else:
                    k += c  # extend key
            elif state == 1:  # reading item value
                if v == '' and c == '"':
                    v += c  # include quote mark in value
                    state = 2  # start reading quoted value
                elif c == ',':
                    val[k] = v  # store parsed item
                    state = 0  # read next key
                    k = ''
                    v = ''
                else:
                    v += c
            elif state == 2:  # reading quoted item value
                if c == '"':
                    v += c  # include quote mark in value
                    state = 1  # end quoting
                else:
                    v += c
        if k != '':
            val[k] = v
        return key, val

    def read_meta(self, meta_string):
        if re.match("##.+=<", meta_string):
            return self.read_meta_hash(meta_string)
        match = self.meta_pattern.match(meta_string)
        if not match:
            # Spec only allows key=value, but we try to be liberal and
            # interpret anything else as key=none (and all values are parsed
            # as strings).
            return meta_string.lstrip('#'), 'none'
        return match.group('key'), match.group('val')


[docs]class Reader(object): """ Reader for a VCF v 4.0 file, an iterator returning ``_Record objects`` """ def __init__(self, fsock=None, filename=None, compressed=None, prepend_chr=False, strict_whitespace=False, encoding='ascii'): """ Create a new Reader for a VCF file. You must specify either fsock (stream) or filename. Gzipped streams or files are attempted to be recogized by the file extension, or gzipped can be forced with ``compressed=True`` 'prepend_chr=True' will put 'chr' before all the CHROM values, useful for different sources. 'strict_whitespace=True' will split records on tabs only (as with VCF spec) which allows you to parse files with spaces in the sample names. """ super(Reader, self).__init__() if not (fsock or filename): raise Exception('You must provide at least fsock or filename') if fsock: self._reader = fsock if filename is None and hasattr(fsock, 'name'): filename = fsock.name if compressed is None: compressed = filename.endswith('.gz') elif filename: if compressed is None: compressed = filename.endswith('.gz') self._reader = open(filename, 'rb' if compressed else 'rt') self.filename = filename if compressed: self._reader = gzip.GzipFile(fileobj=self._reader) if sys.version > '3': self._reader = codecs.getreader(encoding)(self._reader) if strict_whitespace: self._separator = '\t' else: self._separator = '\t| +' self._row_pattern = re.compile(self._separator) self._alt_pattern = re.compile('[\[\]]') self.reader = (line.strip() for line in self._reader if line.strip()) #: metadata fields from header (string or hash, depending) self.metadata = None #: INFO fields from header self.infos = None #: FILTER fields from header self.filters = None #: ALT fields from header self.alts = None #: FORMAT fields from header self.formats = None #: contig fields from header self.contigs = None self.samples = None self._sample_indexes = None self._header_lines = [] self._column_headers = [] self._tabix = None self._prepend_chr = prepend_chr self._parse_metainfo() self._format_cache = {} self.encoding = encoding def __iter__(self): return self def _parse_metainfo(self): '''Parse the information stored in the metainfo of the VCF. The end user shouldn't have to use this. She can access the metainfo directly with ``self.metadata``.''' for attr in ('metadata', 'infos', 'filters', 'alts', 'contigs', 'formats'): setattr(self, attr, OrderedDict()) parser = _vcf_metadata_parser() line = next(self.reader) while line.startswith('##'): self._header_lines.append(line) if line.startswith('##INFO'): key, val = parser.read_info(line) self.infos[key] = val elif line.startswith('##FILTER'): key, val = parser.read_filter(line) self.filters[key] = val elif line.startswith('##ALT'): key, val = parser.read_alt(line) self.alts[key] = val elif line.startswith('##FORMAT'): key, val = parser.read_format(line) self.formats[key] = val elif line.startswith('##contig'): key, val = parser.read_contig(line) self.contigs[key] = val else: key, val = parser.read_meta(line) if key in SINGULAR_METADATA: self.metadata[key] = val else: if key not in self.metadata: self.metadata[key] = [] self.metadata[key].append(val) line = next(self.reader) fields = self._row_pattern.split(line[1:]) self._column_headers = fields[:9] self.samples = fields[9:] self._sample_indexes = dict([(x,i) for (i,x) in enumerate(self.samples)]) def _map(self, func, iterable, bad=['.', '']): '''``map``, but make bad values None.''' return [func(x) if x not in bad else None for x in iterable] def _parse_filter(self, filt_str): '''Parse the FILTER field of a VCF entry into a Python list NOTE: this method has a cython equivalent and care must be taken to keep the two methods equivalent ''' if filt_str == '.': return None elif filt_str == 'PASS': return [] else: return filt_str.split(';') def _parse_info(self, info_str): '''Parse the INFO field of a VCF entry into a dictionary of Python types. ''' if info_str == '.': return {} entries = info_str.split(';') retdict = {} for entry in entries: entry = entry.split('=', 1) ID = entry[0] try: entry_type = self.infos[ID].type except KeyError: try: entry_type = RESERVED_INFO[ID] except KeyError: if entry[1:]: entry_type = 'String' else: entry_type = 'Flag' if entry_type == 'Integer': vals = entry[1].split(',') try: val = self._map(int, vals) # Allow specified integers to be flexibly parsed as floats. # Handles cases with incorrectly specified header types. except ValueError: val = self._map(float, vals) elif entry_type == 'Float': vals = entry[1].split(',') val = self._map(float, vals) elif entry_type == 'Flag': val = True elif entry_type in ('String', 'Character'): try: vals = entry[1].split(',') # commas are reserved characters indicating multiple values val = self._map(str, vals) except IndexError: entry_type = 'Flag' val = True try: if self.infos[ID].num == 1 and entry_type not in ( 'Flag', ): val = val[0] except KeyError: pass retdict[ID] = val return retdict def _parse_sample_format(self, samp_fmt): """ Parse the format of the calls in this _Record """ samp_fmt = make_calldata_tuple(samp_fmt.split(':')) for fmt in samp_fmt._fields: try: entry_type = self.formats[fmt].type entry_num = self.formats[fmt].num except KeyError: entry_num = None try: entry_type = RESERVED_FORMAT[fmt] except KeyError: entry_type = 'String' samp_fmt._types.append(entry_type) samp_fmt._nums.append(entry_num) return samp_fmt def _parse_samples(self, samples, samp_fmt, site): '''Parse a sample entry according to the format specified in the FORMAT column. NOTE: this method has a cython equivalent and care must be taken to keep the two methods equivalent ''' # check whether we already know how to parse this format if samp_fmt not in self._format_cache: self._format_cache[samp_fmt] = self._parse_sample_format(samp_fmt) samp_fmt = self._format_cache[samp_fmt] if cparse: return cparse.parse_samples( self.samples, samples, samp_fmt, samp_fmt._types, samp_fmt._nums, site) samp_data = [] _map = self._map nfields = len(samp_fmt._fields) for name, sample in itertools.izip(self.samples, samples): # parse the data for this sample sampdat = [None] * nfields for i, vals in enumerate(sample.split(':')): # short circuit the most common if samp_fmt._fields[i] == 'GT': sampdat[i] = vals continue # genotype filters are a special case elif samp_fmt._fields[i] == 'FT': sampdat[i] = self._parse_filter(vals) continue elif not vals or vals == ".": sampdat[i] = None continue entry_num = samp_fmt._nums[i] entry_type = samp_fmt._types[i] # we don't need to split single entries if entry_num == 1: if entry_type == 'Integer': try: sampdat[i] = int(vals) except ValueError: sampdat[i] = float(vals) elif entry_type == 'Float' or entry_type == 'Numeric': sampdat[i] = float(vals) else: sampdat[i] = vals continue vals = vals.split(',') if entry_type == 'Integer': try: sampdat[i] = _map(int, vals) except ValueError: sampdat[i] = _map(float, vals) elif entry_type == 'Float' or entry_type == 'Numeric': sampdat[i] = _map(float, vals) else: sampdat[i] = vals # create a call object call = _Call(site, name, samp_fmt(*sampdat)) samp_data.append(call) return samp_data def _parse_alt(self, str): if self._alt_pattern.search(str) is not None: # Paired breakend items = self._alt_pattern.split(str) remoteCoords = items[1].split(':') chr = remoteCoords[0] if chr[0] == '<': chr = chr[1:-1] withinMainAssembly = False else: withinMainAssembly = True pos = remoteCoords[1] orientation = (str[0] == '[' or str[0] == ']') remoteOrientation = (re.search('\[', str) is not None) if orientation: connectingSequence = items[2] else: connectingSequence = items[0] return _Breakend(chr, pos, orientation, remoteOrientation, connectingSequence, withinMainAssembly) elif str[0] == '.' and len(str) > 1: return _SingleBreakend(True, str[1:]) elif str[-1] == '.' and len(str) > 1: return _SingleBreakend(False, str[:-1]) elif str[0] == "<" and str[-1] == ">": return _SV(str[1:-1]) else: return _Substitution(str)
[docs] def next(self): '''Return the next record in the file.''' line = next(self.reader) row = self._row_pattern.split(line.rstrip()) chrom = row[0] if self._prepend_chr: chrom = 'chr' + chrom pos = int(row[1]) if row[2] != '.': ID = row[2] else: ID = None ref = row[3] alt = self._map(self._parse_alt, row[4].split(',')) try: qual = int(row[5]) except ValueError: try: qual = float(row[5]) except ValueError: qual = None filt = self._parse_filter(row[6]) info = self._parse_info(row[7]) try: fmt = row[8] except IndexError: fmt = None else: if fmt == '.': fmt = None record = _Record(chrom, pos, ID, ref, alt, qual, filt, info, fmt, self._sample_indexes) if fmt is not None: samples = self._parse_samples(row[9:], fmt, record) record.samples = samples return record
[docs] def fetch(self, chrom, start=None, end=None): """ Fetches records from a tabix-indexed VCF file and returns an iterable of ``_Record`` instances chrom must be specified. The start and end coordinates are in the zero-based, half-open coordinate system, similar to ``_Record.start`` and ``_Record.end``. The very first base of a chromosome is index 0, and the the region includes bases up to, but not including the base at the end coordinate. For example ``fetch('4', 10, 20)`` would include all variants overlapping a 10 base pair region from the 11th base of through the 20th base (which is at index 19) of chromosome 4. It would not include the 21st base (at index 20). See http://genomewiki.ucsc.edu/index.php/Coordinate_Transforms for more information on the zero-based, half-open coordinate system. If end is omitted, all variants from start until the end of the chromosome chrom will be included. If start and end are omitted, all variants on chrom will be returned. requires pysam """ if not pysam: raise Exception('pysam not available, try "pip install pysam"?') if not self.filename: raise Exception('Please provide a filename (or a "normal" fsock)') if not self._tabix: self._tabix = pysam.Tabixfile(self.filename, encoding=self.encoding) if self._prepend_chr and chrom[:3] == 'chr': chrom = chrom[3:] self.reader = self._tabix.fetch(chrom, start, end) return self
[docs]class Writer(object): """VCF Writer. On Windows Python 2, open stream with 'wb'.""" # Reverse keys and values in header field count dictionary counts = dict((v,k) for k,v in field_counts.iteritems()) def __init__(self, stream, template, lineterminator="\n"): self.writer = csv.writer(stream, delimiter="\t", lineterminator=lineterminator, quotechar='', quoting=csv.QUOTE_NONE) self.template = template self.stream = stream # Order keys for INFO fields defined in the header (undefined fields # get a maximum key). self.info_order = collections.defaultdict( lambda: len(template.infos), dict(zip(template.infos.iterkeys(), itertools.count()))) two = '##{key}=<ID={0},Description="{1}">\n' four = '##{key}=<ID={0},Number={num},Type={2},Description="{3}">\n' _num = self._fix_field_count for (key, vals) in template.metadata.iteritems(): if key in SINGULAR_METADATA: vals = [vals] for val in vals: if isinstance(val, dict): values = ','.join('{0}={1}'.format(key, value) for key, value in val.items()) stream.write('##{0}=<{1}>\n'.format(key, values)) else: stream.write('##{0}={1}\n'.format(key, val)) for line in template.infos.itervalues(): stream.write(four.format(key="INFO", *line, num=_num(line.num))) for line in template.formats.itervalues(): stream.write(four.format(key="FORMAT", *line, num=_num(line.num))) for line in template.filters.itervalues(): stream.write(two.format(key="FILTER", *line)) for line in template.alts.itervalues(): stream.write(two.format(key="ALT", *line)) for line in template.contigs.itervalues(): if line.length: stream.write('##contig=<ID={0},length={1}>\n'.format(*line)) else: stream.write('##contig=<ID={0}>\n'.format(*line)) self._write_header() def _write_header(self): # TODO: write INFO, etc self.stream.write('#' + '\t'.join(self.template._column_headers + self.template.samples) + '\n')
[docs] def write_record(self, record): """ write a record to the file """ ffs = self._map(str, [record.CHROM, record.POS, record.ID, record.REF]) \ + [self._format_alt(record.ALT), record.QUAL or '.', self._format_filter(record.FILTER), self._format_info(record.INFO)] if record.FORMAT: ffs.append(record.FORMAT) samples = [self._format_sample(record.FORMAT, sample) for sample in record.samples] self.writer.writerow(ffs + samples)
[docs] def flush(self): """Flush the writer""" try: self.stream.flush() except AttributeError: pass
[docs] def close(self): """Close the writer""" try: self.stream.close() except AttributeError: pass
def _fix_field_count(self, num_str): """Restore header number to original state""" if num_str not in self.counts: return num_str else: return self.counts[num_str] def _format_alt(self, alt): return ','.join(self._map(str, alt)) def _format_filter(self, flt): if flt == []: return 'PASS' return self._stringify(flt, none='.', delim=';') def _format_info(self, info): if not info: return '.' def order_key(field): # Order by header definition first, alphabetically second. return self.info_order[field], field return ';'.join(self._stringify_pair(f, info[f]) for f in sorted(info, key=order_key)) def _format_sample(self, fmt, sample): if hasattr(sample.data, 'GT'): gt = sample.data.GT else: gt = './.' if 'GT' in fmt else '' result = [gt] if gt else [] # Following the VCF spec, GT is always the first item whenever it is present. for field in sample.data._fields: value = getattr(sample.data,field) if field == 'GT': continue if field == 'FT': result.append(self._format_filter(value)) else: result.append(self._stringify(value)) return ':'.join(result) def _stringify(self, x, none='.', delim=','): if type(x) == type([]): return delim.join(self._map(str, x, none)) return str(x) if x is not None else none def _stringify_pair(self, x, y, none='.', delim=','): if isinstance(y, bool): return str(x) if y else "" return "%s=%s" % (str(x), self._stringify(y, none=none, delim=delim)) def _map(self, func, iterable, none='.'): '''``map``, but make None values none.''' return [func(x) if x is not None else none for x in iterable]
def __update_readme(): import sys, vcf file('README.rst', 'w').write(vcf.__doc__) # backwards compatibility VCFReader = Reader VCFWriter = Writer