# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import io
import locale
import logging
import os
import re
import sys

from tailhead.version import VERSION as __version__

if sys.version_info < (3,):
    range = xrange

try:
    from io import SEEK_SET, SEEK_CUR, SEEK_END
except:
    SEEK_SET = 0
    SEEK_CUR = 1
    SEEK_END = 2

LOG = logging.getLogger('tailhead')

class Tailer(object):
    """
    Implements tailing and heading functionality like GNU tail and head
    commands.
    """
    LINE_TERMINATORS = (b'\r\n', b'\n', b'\r')

    def __init__(self, file, read_size=1024, end=False):
        """
        Tailer requires file to be opened in binary mode, otherwise
        the behavior of tell suffers from implementation detail on Windows
        where it may return negative numbers. Per modern Python documentation
        tell returns opaque number for text objects that may not represent
        number of bytes.

        :param file: File-like object opened in binary mode.
        :param read_size: How many bytes to read at once. Affects head and tail calls.
        :param end: Whether to start reading from end.

        :raise ValueError: If file is not open in binary mode.
        """
        if not isinstance(file, io.IOBase) or isinstance(file, io.TextIOBase):
            raise ValueError("io object must be in the binary mode")

        self.read_size = read_size
        self.file = file

        if end:
            self.file.seek(0, SEEK_END)

    def splitlines(self, data):
        """
        Split data into lines where lines are separated by LINE_TERMINATORS.

        :param data: Any chunk of binary data.
        :return: List of lines without any characters at LINE_TERMINATORS.
        """
        return re.split(b'|'.join(self.LINE_TERMINATORS), data)

    def read(self, read_size=-1):
        """
        Read given number of bytes from file.
        :param read_size: Number of bytes to read. -1 to read all.
        :return: Number of bytes read and data that was read.
        """
        read_str = self.file.read(read_size)
        return len(read_str), read_str

    def prefix_line_terminator(self, data):
        """
        Return line terminator data begins with or None.
        """
        for t in self.LINE_TERMINATORS:
            if data.startswith(t):
                return t

        return None

    def suffix_line_terminator(self, data):
        """
        Return line terminator data ends with or None.
        """
        for t in self.LINE_TERMINATORS:
            if data.endswith(t):
                return t

        return None

    def seek_next_line(self):
        """
        Seek next line relative to the current file position.

        :return: Position of the line or -1 if next line was not found.
        """
        where = self.file.tell()
        offset = 0

        while True:
            data_len, data = self.read(self.read_size)
            data_where = 0

            if not data_len:
                break

            # Consider the following example: Foo\r | \nBar where " | " denotes current position,
            # 'Foo\r' is the read part and '\nBar' is the remaining part.
            # We should completely consume terminator "\r\n" by reading one extra byte.
            if b'\r\n' in self.LINE_TERMINATORS and data[-1] == b'\r'[0]:
                terminator_where = self.file.tell()
                terminator_len, terminator_data = self.read(1)

                if terminator_len and terminator_data[0] == b'\n'[0]:
                    data_len += 1
                    data += b'\n'
                else:
                    self.file.seek(terminator_where)

            while data_where < data_len:
                terminator = self.prefix_line_terminator(data[data_where:])
                if terminator:
                    self.file.seek(where + offset + data_where + len(terminator))
                    return self.file.tell()
                else:
                    data_where += 1

            offset += data_len
            self.file.seek(where + offset)

        return -1

    def seek_previous_line(self):
        """
        Seek previous line relative to the current file position.

        :return: Position of the line or -1 if previous line was not found.
        """
        where = self.file.tell()
        offset = 0

        while True:
            if offset == where:
                break

            read_size = self.read_size if self.read_size <= where else where
            self.file.seek(where - offset - read_size, SEEK_SET)
            data_len, data = self.read(read_size)

            # Consider the following example: Foo\r | \nBar where " | " denotes current position,
            # '\nBar' is the read part and 'Foo\r' is the remaining part.
            # We should completely consume terminator "\r\n" by reading one extra byte.
            if b'\r\n' in self.LINE_TERMINATORS and data[0] == b'\n'[0]:
                terminator_where = self.file.tell()
                if terminator_where > data_len + 1:
                    self.file.seek(where - offset - data_len - 1, SEEK_SET)
                    terminator_len, terminator_data = self.read(1)

                    if terminator_data[0] == b'\r'[0]:
                        data_len += 1
                        data = b'\r' + data

                    self.file.seek(terminator_where)

            data_where = data_len

            while data_where > 0:
                terminator = self.suffix_line_terminator(data[:data_where])
                if terminator and offset == 0 and data_where == data_len:
                    # The last character is a line terminator that finishes current line. Ignore it.
                    data_where -= len(terminator)
                elif terminator:
                    self.file.seek(where - offset - (data_len - data_where))
                    return self.file.tell()
                else:
                    data_where -= 1

            offset += data_len

        if where == 0:
            # Nothing more to read.
            return -1
        else:
            # Very first line.
            self.file.seek(0)
            return 0

    def tail(self, lines=10):
        """
        Return the last lines of the file.
        """
        self.file.seek(0, SEEK_END)

        for i in range(lines):
            if self.seek_previous_line() == -1:
                break

        data = self.file.read()

        for t in self.LINE_TERMINATORS:
            if data.endswith(t):
                # Only terminators _between_ lines should be preserved.
                # Otherwise terminator of the last line will be treated as separtaing line and empty line.
                data = data[:-len(t)]
                break

        if data:
            return self.splitlines(data)
        else:
            return []

    def head(self, lines=10):
        """
        Return the top lines of the file.
        """
        self.file.seek(0)

        for i in range(lines):
            if self.seek_next_line() == -1:
                break

        end_pos = self.file.tell()

        self.file.seek(0)
        data = self.file.read(end_pos)

        for t in self.LINE_TERMINATORS:
            if data.endswith(t):
                # Only terminators _between_ lines should be preserved.
                # Otherwise terminator of the last line will be treated as separtaing line and empty line.
                data = data[:-len(t)]
                break

        if data:
            return self.splitlines(data)
        else:
            return []

    def follow(self):
        """
        Iterator generator that returns lines as data is added to the file.

        None will be yielded if no new line is available.
        Caller may either wait and re-try or end iteration.
        """
        trailing = True

        while True:
            where = self.file.tell()

            if where > os.fstat(self.file.fileno()).st_size:
                # File was truncated.
                where = 0
                self.file.seek(where)

            line = self.file.readline()

            if line:
                if trailing and line in self.LINE_TERMINATORS:
                    # This is just the line terminator added to the end of the file
                    # before a new line, ignore.
                    trailing = False
                    continue

                terminator = self.suffix_line_terminator(line)
                if terminator:
                    line = line[:-len(terminator)]

                trailing = False
                yield line
            else:
                trailing = True
                self.file.seek(where)
                yield None

def tail(file, lines=10, read_size=1024):
    """
    Return the last lines of the file.

    >>> import io
    >>>
    ... with io.open('test_tail.txt', 'w+') as fw:
    ...     with io.open('test_tail.txt', 'rb') as fr:
    ...         _ = fw.write('\\r')
    ...         _ = fw.write('Line 1\\r\\n')
    ...         _ = fw.write('Line 2\\n')
    ...         _ = fw.write('Line 3\\r')
    ...         _ = fw.write('Line 4\\r\\n')
    ...         _ = fw.write('\\n')
    ...         _ = fw.write('\\r\\n')
    ...         _ = fw.write('\\r\\n')
    ...         fw.flush()
    ...         tail(fr, 6, 1)  # doctest: +ELLIPSIS
    [...'Line 2', ...'Line 3', ...'Line 4', ...'', ...'', ...'']
    >>> os.remove('test_tail.txt')

    >>> import io
    >>>
    ... with io.open('test_tail.txt', 'w+') as fw:
    ...     with io.open('test_tail.txt', 'rb') as fr:
    ...         _ = fw.write('Line 1')
    ...         fw.flush()
    ...         tail(fr, 6, 1)  # doctest: +ELLIPSIS
    [...'Line 1']
    >>> os.remove('test_tail.txt')
    """
    return Tailer(file, read_size).tail(lines)

def head(file, lines=10, read_size=1024):
    """
    Return the top lines of the file.

    >>> import io
    >>>
    ... with io.open('test_head.txt', 'w+') as fw:
    ...     with io.open('test_head.txt', 'rb') as fr:
    ...         _ = fw.write('\\r\\n')
    ...         _ = fw.write('\\r\\n')
    ...         _ = fw.write('\\r')
    ...         _ = fw.write('Line 1\\r\\n')
    ...         _ = fw.write('Line 2\\r\\n')
    ...         _ = fw.write('Line 3\\r')
    ...         _ = fw.write('Line 4\\r\\n')
    ...         _ = fw.write('\\n')
    ...         _ = fw.write('\\r')
    ...         fw.flush()
    ...         head(fr, 6, 1)  # doctest: +ELLIPSIS
    [...'', ...'', ...'', ...'Line 1', ...'Line 2', ...'Line 3']
    >>> os.remove('test_head.txt')
    """
    return Tailer(file, read_size).head(lines)

def follow(file):
    """
    Generator that returns lines as data is added to the file.

    Returned generator yields bytes.

    >>> import io
    >>> import os
    >>> f = io.open('test_follow.txt', 'w+')
    >>> fo = io.open('test_follow.txt', 'rb')
    >>> generator = follow(fo)
    >>> _ = f.write('Line 1\\n')
    >>> f.flush()
    >>> print(next(generator).decode('utf-8'))
    Line 1
    >>> _ = f.write('Line 2\\n')
    >>> f.flush()
    >>> print(next(generator).decode('utf-8'))
    Line 2
    >>> _ = f.truncate(0)
    >>> _ = f.seek(0)
    >>> _ = f.write('Line 3\\n')
    >>> f.flush()
    >>> print(next(generator).decode('utf-8'))
    Line 3
    >>> print(next(generator))
    None
    >>> f.close()
    >>> fo.close()
    >>> os.remove('test_follow.txt')
    """
    return Tailer(file, end=True).follow()

def follow_path(file_path, buffering=-1, encoding=None, errors='strict'):
    """
    Similar to follow, but also looks up if inode of file is changed
    e.g. if it was re-created.

    Returned generator yields strings encoded by using encoding.
    If encoding is not specified, it defaults to locale.getpreferredencoding()

    >>> import io
    >>> import os
    >>> f = io.open('test_follow_path.txt', 'w+')
    >>> generator = follow_path('test_follow_path.txt')
    >>> _ = f.write('Line 1\\n')
    >>> f.flush()
    >>> print(next(generator))
    Line 1
    >>> _ = f.write('Line 2\\n')
    >>> f.flush()
    >>> print(next(generator))
    Line 2
    >>> _ = f.truncate(0)
    >>> _ = f.seek(0)
    >>> _ = f.write('Line 3\\n')
    >>> f.flush()
    >>> print(next(generator))
    Line 3
    >>> f.close()
    >>> os.remove('test_follow_path.txt')
    >>> f = io.open('test_follow_path.txt', 'w+')
    >>> _ = f.write('Line 4\\n')
    >>> f.flush()
    >>> print(next(generator))
    Line 4
    >>> print(next(generator))
    None
    >>> f.close()
    >>> os.remove('test_follow_path.txt')
    """
    if encoding is None:
        encoding = locale.getpreferredencoding()

    class FollowPathGenerator(object):
        def __init__(self):
            if os.path.isfile(file_path):
                self.following_file = io.open(file_path, 'rb', buffering)
                self.follow_generator = Tailer(self.following_file, end=True).follow()
                self.follow_from_end_on_open = False
            else:
                self.following_file = None
                self.follow_generator = None
                self.follow_from_end_on_open = True

        def next(self):
            while True:
                if self.follow_generator:
                    line = next(self.follow_generator)
                else:
                    line = None

                if line is None:
                    if self.follow_generator:
                        try:
                            is_file_changed = not os.path.isfile(file_path) or os.stat(file_path).st_ino != os.fstat(self.following_file.fileno()).st_ino
                        except OSError:
                            # File could be deleted between isfile and stat invocations, which will make the latter to fail.
                            is_file_changed = True

                        if is_file_changed:
                            # File was deleted or re-created.
                            self.following_file.close()
                            self.following_file = None
                            self.follow_generator = None

                    if not self.follow_generator and os.path.isfile(file_path):
                        # New file is available. Open it.
                        try:
                            self.following_file = io.open(file_path, 'rb', buffering)
                            self.follow_generator = Tailer(self.following_file, end=self.follow_from_end_on_open).follow()
                            self.follow_from_end_on_open = False  # something could be written before we noticed change of file
                        except (IOError, OSError) as e:
                            LOG.info("Unable to tail file: %s", e)
                            if self.following_file:
                                self.following_file.close()

                            self.following_file= None
                            self.follow_generator = None
                            line = None
                        else:
                            line = next(self.follow_generator)

                return line.decode(encoding, errors) if line is not None else line

        def __iter__(self):
            return self

        def __next__(self):
            return self.next()

    return FollowPathGenerator()


"""Interprets Strelka VCF files adding Jacquard standard information.

* Strelka VCFs are assumed to have a ".vcf" extension and have a
    "##source=strelka" metaheader.
* Strelka produces a separate file for SNVs and indels. Jacquard can process
    either or both.
* Jacquard standard tags are based on tier 2 data and use different source tags
    based on whether the file is indel or snp.

See tag definitions for more info.
"""
from __future__ import print_function, absolute_import, division

from collections import defaultdict, OrderedDict
import os
import re

import jacquard.utils.logger as logger
import jacquard.utils.utils as utils
import jacquard.variant_caller_transforms.common_tags as common_tags
import jacquard.utils.vcf as vcf

JQ_STRELKA_TAG = "JQ_SK_"
STRELKA_ABBREVIATION = "SK"
VERSION = "v1.0 - v2.9"

class _GenotypeTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods

    _INDEL_VALUES = ["ref", "hom", "het"]

    def __init__(self):
        super(self.__class__,
              self).__init__(STRELKA_ABBREVIATION,
                             common_tags.GENOTYPE_TAG,
                             ('Jacquard genotype (based on SGT). Example for '
                              'snv: REF=A, ALT=C, INFO:SGT=AA->AC is translated'
                              ' as normal=0/0, tumor=0/1. Example for indel: '
                              'INFO:SGT=ref->het is translated as normal=0/0, '
                              'tumor=0/1.'))

    @staticmethod
    def _get_indel_genotype(sample_genotype):
        indel_mapping_values = {"ref":"0/0","het":"0/1","hom":"1/1"}
        if sample_genotype in indel_mapping_values:
            return indel_mapping_values[sample_genotype]
        else:
            raise utils.JQException("Unable to determine Genotype")

    @staticmethod
    def _get_snv_genotype(sample_genotype, ref, alt):
        sample_allele1, sample_allele2 = list(sample_genotype)
        alleles = [ref]
        alleles.extend(alt.split(","))

        chrom_a = None
        chrom_b = None

        for i, allele in enumerate(alleles):
            if sample_allele1 == allele:
                chrom_a = str(i)
            if sample_allele2 == allele:
                chrom_b = str(i)

        if chrom_a and chrom_b:
            return "/".join(sorted([chrom_a, chrom_b]))
        else:
            raise utils.JQException("Unable to determine Genotype")

    def add_tag_values(self, vcf_record):
        sample_values = {}
        if "SGT" in vcf_record.info_dict:
            sample_genotypes = vcf_record.info_dict["SGT"].split("->")

            for i, sample in enumerate(vcf_record.sample_tag_values):
                if sample_genotypes[i] in _GenotypeTag._INDEL_VALUES:
                    genotype = self._get_indel_genotype(sample_genotypes[i])
                else:
                    genotype = self._get_snv_genotype(sample_genotypes[i],
                                                      vcf_record.ref,
                                                      vcf_record.alt)
                sample_values[sample] = genotype

        if sample_values:
            vcf_record.add_sample_tag_value(self.tag_id,
                                            sample_values)

class _AlleleFreqTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(STRELKA_ABBREVIATION,
                             common_tags.ALLELE_FREQ_TAG,
                             ('Jacquard allele frequency for Strelka: Decimal '
                              'allele frequency rounded to 4 digits (based on '
                              'alt_depth/total_depth. Uses (TIR tier 2)/DP2 if '
                              'available, otherwise uses (ACGT tier2 depth) / '
                              'DP2)'))

    @staticmethod
    def _get_tier2_base_depth(sample_format_dict, alt_allele):
        numerator = float(sample_format_dict[alt_allele + "U"].split(",")[1])
        tags = ["AU", "CU", "TU", "GU"]
        depth = 0

        for tag in tags:
            depth += float(sample_format_dict[tag].split(",")[1])
        freq = numerator/depth if depth != 0 else 0.0

        return freq

    def _get_snp_allele_freq_per_sample(self, vcf_record, sample):
        afs = []
        split_alt = vcf_record.alt.split(",")

        for alt_allele in split_alt:
            sample_format_dict = vcf_record.sample_tag_values[sample]
            freq = self._get_tier2_base_depth(sample_format_dict, alt_allele)

            rounded_af = self._standardize_af(str(freq))
            capped_af = min(rounded_af, "1.00")
            afs.append(capped_af)

        return afs

    def _get_indelallelefreq_per_sample(self, vcf_record, sample):
        afs = []
        indel_depths = vcf_record.sample_tag_values[sample]["TIR"]
        numerator = float(indel_depths.split(",")[1])
        denominator = float(vcf_record.sample_tag_values[sample]["DP2"])
        freq = numerator/denominator if denominator != 0 else 0.0

        rounded_af = self._standardize_af(str(freq))
        capped_af = min(rounded_af, "1.00")
        afs.append(capped_af)

        return afs

    def add_tag_values(self, vcf_record):
        sample_values = {}
        if vcf_record.alt == ".":
            for sample in vcf_record.sample_tag_values:
                sample_values[sample] = "."
        else:
            for sample in vcf_record.sample_tag_values:
                if "AU" in vcf_record.format_tags:#if it's an snp
                    afs = self._get_snp_allele_freq_per_sample(vcf_record,
                                                               sample)
                    sample_values[sample] = ",".join(afs)
                elif "TIR" in vcf_record.format_tags: #if it's an indel
                    afs = self._get_indelallelefreq_per_sample(vcf_record,
                                                               sample)
                    sample_values[sample] = ",".join(afs)
                else:
                    continue

        if sample_values:
            vcf_record.add_sample_tag_value(self.tag_id,
                                            sample_values)

    @staticmethod
    def _standardize_af(value):
        return utils.round_digits(value)

class _DepthTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    REQUIRED_TAGS = set(["DP2", "AU"])
    NUCLEOTIDE_DEPTH_TAGS = ["AU", "CU", "TU", "GU"]

    @classmethod
    def _get_tier2_base_depth(cls, sample_tags):
        if "DP2" in sample_tags:
            return sample_tags["DP2"]
        depth = 0
        for tag in _DepthTag.NUCLEOTIDE_DEPTH_TAGS:
            tier2_depth = sample_tags[tag].split(",")[1]
            depth += int(tier2_depth)
        return str(depth)

    def __init__(self):
        super(self.__class__,
              self).__init__(STRELKA_ABBREVIATION,
                             common_tags.DEPTH_TAG,
                             ('Jacquard depth for Strelka (uses DP2 if '
                              'available, otherwise uses ACGT tier2 depth)'))

    def add_tag_values(self, vcf_record):
        if vcf_record.format_tags.isdisjoint(_DepthTag.REQUIRED_TAGS):
            return
        sample_values = {}
        for sample in vcf_record.sample_tag_values:
            sample_tags = vcf_record.sample_tag_values[sample]
            sample_values[sample] = _DepthTag._get_tier2_base_depth(sample_tags)
        vcf_record.add_sample_tag_value(self.tag_id, sample_values)

##TODO (cgates): Make this robust to sample order changes
class _SomaticTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(STRELKA_ABBREVIATION,
                             common_tags.SOMATIC_TAG,
                             ('Jacquard somatic status for Strelka: '
                              '0=non-somatic,1=somatic (based on PASS in FILTER'
                              ' column)'))

    def add_tag_values(self,vcf_record):
        sample_values = {}
        for i, sample in enumerate(vcf_record.sample_tag_values):
            sample_values[sample] = _SomaticTag._somatic_status(i, vcf_record)
        vcf_record.add_sample_tag_value(self.tag_id, sample_values)

    @classmethod
    def _somatic_status(cls, sample_index, vcf_record):
        if sample_index == 1 and vcf_record.filter == "PASS":
            return "1"
        else:
            return "0"

class Strelka(object):
    #pylint: disable=too-few-public-methods
    """Recognize and transform Strelka VCFs to standard Jacquard format.

    Note that Strelka sometimes reports variants which fail the filter as
    having an ALT of "."; this seems to be Strelka's shorthand for saying,
    "I couldn't be sure there was an ALT". Unfortunately, that's not a valid
    ALT value so these rows are flagged and (in a typical workflow) excluded.
    """

    def __init__(self):
        self.name = "Strelka"
        self.abbr = "SK"
        self.meta_header = "##jacquard.normalize_strelka.sources={0},{1}\n"

    @staticmethod
    def _is_strelka_vcf(file_reader):
        if file_reader.file_name.endswith(".vcf"):
            vcf_reader = vcf.VcfReader(file_reader)
            return "##source=strelka" in vcf_reader.metaheaders
        return False

    def _find_strelka_files(self, file_readers):
        prefix_dict = OrderedDict()
        unclaimed_readers = []
        for file_reader in file_readers:
            if self._is_strelka_vcf(file_reader):
                prefix, _ = os.path.splitext(file_reader.file_name)
                prefix_dict[prefix] = file_reader
            else:
                unclaimed_readers.append(file_reader)

        return prefix_dict, unclaimed_readers

    @staticmethod
    def _split_prefix_by_patient(prefix_dict):
        prefix_by_patients = defaultdict(list)
        snv_indel = ["snvs", "indels"]

        for prefix in prefix_dict:
            patient_names = [i for i in prefix.split(".") if i not in snv_indel]
            patient_name = ".".join(patient_names)
            prefix_by_patients[patient_name].append(prefix)

        return prefix_by_patients

    @staticmethod
    def _validate_vcf_readers(prefix_by_patients):
        number_of_files = set()
        for file_names in prefix_by_patients.values():
            if len(file_names) == 1:
                for file_name in file_names:
                    if re.search("snvs", file_name):
                        msg = "Strelka VCF [{}] has no indels file."
                        logger.error(msg, file_name)
                    elif re.search("indels", file_name):
                        msg = "Strelka VCF [{}] has no snvs file."
                        logger.error(msg, file_name)
            number_of_files.add(len(file_names))

        if len(number_of_files) > 1:
            msg = ("Some Strelka VCFs were missing either a snvs or indels "
                   "file. Review inputs/command options and try again.")
            raise utils.JQException(msg)

    @staticmethod
    def _create_vcf_readers(prefix_to_readers):
        vcf_readers = []
        for file_reader in prefix_to_readers.values():
            vcf_reader = vcf.VcfReader(file_reader)
            vcf_readers.append(_StrelkaVcfReader(vcf_reader))

        return vcf_readers

    def claim(self, file_readers):
        """Recognizes and claims Strelka VCFs form the set of all input VCFs.

        Each defined caller has a chance to evaluate and claim all the incoming
        files as something that it can process.

        Args:
            file_readers: the collection of currently unclaimed files

        Returns:
            A tuple of unclaimed readers and StrelkaVcfReaders.
        """
        (prefix_to_reader,
         unclaimed_readers) = self._find_strelka_files(file_readers)
        prefix_by_patients = self._split_prefix_by_patient(prefix_to_reader)
        self._validate_vcf_readers(prefix_by_patients)
        vcf_readers = self._create_vcf_readers(prefix_to_reader)

        return (unclaimed_readers, vcf_readers)

class _StrelkaVcfReader(object):
    """Adapter that presents a Strelka VCF as a VcfReader.

    This follows the VcfReader interface, delegating calls to the base
    VcfReader, adjusting metaheaders, and individual
    variants as appropriate.

    See VcfReader for more info.
    """
    def __init__(self, vcf_reader):
        self._vcf_reader = vcf_reader
        self._caller = Strelka()
        self.tags = [common_tags.ReportedTag(STRELKA_ABBREVIATION),
                     common_tags.PassedTag(STRELKA_ABBREVIATION),
                     _AlleleFreqTag(),
                     _DepthTag(),
                     _SomaticTag(),
                     _GenotypeTag()]

    def _get_new_metaheaders(self):
        return [tag.metaheader for tag in self.tags]

    @property
    def caller_name(self):
        return self._caller.name

    @property
    def file_name(self):
        return self._vcf_reader.file_name

    def open(self):
        return self._vcf_reader.open()

    def close(self):
        return self._vcf_reader.close()

    @staticmethod
    def expected_file_format():
        return ["snvs", "indels"]

    @property
    def metaheaders(self):
        new_metaheaders = list(self._vcf_reader.metaheaders)
        new_metaheaders.extend(self._get_new_metaheaders())
        caller_metaheader = "##jacquard.translate.caller={0}".\
                format(self._caller.name)
        new_metaheaders.append(caller_metaheader)

        return new_metaheaders

    @property
    def column_header(self):
        return self._vcf_reader.column_header

    def tagged_vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield vcf_record

    def vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield self._add_tags(vcf_record)

    def _add_tags(self, vcf_record):
        for tag in self.tags:
            tag.add_tag_values(vcf_record)
        return vcf_record


"""Classes to parse, interpret, and manipulate VCF files and records."""
from __future__ import print_function, absolute_import, division

from collections import OrderedDict
import os
import re
import sys

import natsort

import jacquard.utils.utils as utils

#TODO: (cgates): add context management to open/close
class VcfReader(object):
    """Read only wrapper providing VCF metaheaders and records."""
    def __init__(self, file_reader):
        self._file_reader = file_reader
        (self.column_header, self.metaheaders) = self._init_headers()
        self.split_column_header = self.column_header.strip("#").split("\t")
        self.sample_names = self._init_sample_names()
        self.qualified_sample_names = self._create_qualified_sample_names()

    def __lt__(self, other):
        return self._file_reader < other._file_reader

    def _get_tag_metaheaders(self, regex_exp):
        tag_dict = {}
        for metaheader in self.metaheaders:
            tag = re.match(regex_exp, metaheader)
            if tag:
                tag_key = tag.group(1)
                tag_dict[tag_key] = metaheader.strip()

        return tag_dict

    @property
    def file_name(self):
        return self._file_reader.file_name

    @property
    def input_filepath(self):
        return self._file_reader.input_filepath

    @property
    def format_metaheaders(self):
        return dict(self._get_tag_metaheaders("^##FORMAT=.*?[<,]ID=([^,>]*)"))

    @property
    def info_metaheaders(self):
        return dict(self._get_tag_metaheaders("^##INFO=.*?[<,]ID=([^,>]*)"))

    @property
    def filter_metaheaders(self):
        return dict(self._get_tag_metaheaders("^##FILTER=.*?[<,]ID=([^,>]*)"))

    @property
    def contig_metaheaders(self):
        return dict(self._get_tag_metaheaders("^##contig=.*?[<,]ID=([^,>]*)"))

    def _init_sample_names(self):
        sample_names = []
        column_fields = self.column_header.split("\t")
        if len(column_fields) > 8:
            sample_names = column_fields[9:]

        return sample_names

    def _create_qualified_sample_names(self):
        patient_prefix = self.file_name.split(".")[0]
        qualified_names = []
        for sample_name in self.sample_names:
            qualified_names.append("|".join([patient_prefix, sample_name]))
        return qualified_names

    def _init_headers(self):
        metaheaders = []
        column_header = None
        try:
            self._file_reader.open()
            for line in self._file_reader.read_lines():
                if line.startswith("##"):
                    metaheaders.append(line.rstrip())
                elif line.startswith("#"):
                    column_header = line.rstrip()
                else:
                    break
        finally:
            self._file_reader.close()

        if not (column_header and metaheaders):
            raise utils.JQException("ERROR: [{}] is not a valid vcf. Missing "
                                    "column header or metaheaders."\
                                    .format(self.file_name))

#         return column_header, tuple(metaheaders)
        return column_header, metaheaders

    #TODO (cgates): qualified is used by ONE invocation in merge. Can we
    #somehow make merge do this instead of universally complicating the method?
    def vcf_records(self, qualified=False):
        """Generates parsed VcfRecord objects.

        Typically called in a for loop to process each vcf record in a
        VcfReader. VcfReader must be opened in advanced and closed when
        complete. Skips all headers.

        Args:
            qualified: When True, sample names are prefixed with file name

        Returns:
            Parsed VcfRecord

        Raises:
            StopIteration: when reader is exhausted.
            TypeError: if reader is closed.
        """
        if qualified:
            sample_names = self.qualified_sample_names
        else:
            sample_names = self.sample_names

        for line in self._file_reader.read_lines():
            if line.startswith("#"):
                continue
            yield VcfRecord.parse_record(line, sample_names)

    def open(self):
        self._file_reader.open()

    def close(self):
        self._file_reader.close()

class VcfRecord(object): #pylint: disable=too-many-instance-attributes
    """Represents an specific variant record.

    All functionality for parsing, interpreting, or manipulating a VCF record
    should be contained in this class. Mutator methods are implemented to add
    fields/tags to existing record, but replace/remove fields/tags.

    Attributes and args typically mirror the VCF fields; a few are renamed to
    avoid collisions with Python-reserved words. Except where noted, all
    attributes are stored as strings.

    Attributes:
            chrom:
            pos:
            ref:
            alt:
            vcf_id:
            qual:
            vcf_filter:
            info_dict: dict of info fields key-value pairs (flags stored as
                key-None)
            sample_tag_values: dict of samples where each value is a dict of
                tag-values. All keys and values are stored as strings. Example:
                {sampleA : {AF:0.3, DP:42}, sampleB : {AF:0.1, DP:25}}
                Note that dict is a natural pythonic representation of sample
                tag values, it's sometimes helpful to think of sample_tag_values
                as a table of samples (rows) x format_tags (columns).
            _key: tuple that defines record equality and ordering.
    """
    _EMPTY_SET = set()
    _FILTERS_TO_REPLACE = set(["", ".", "pass"])

    @classmethod
    def parse_record(cls, vcf_line, sample_names):
        """Alternative constructor that parses VcfRecord from VCF string.

        Aspire to parse/represent the data such that it could be reliably
        round-tripped. (This nicety means INFO fields and FORMAT tags should be
        treated as ordered to avoid shuffling.)

        Args:
            vcf_line: the VCF variant record as a string; tab separated fields,
                trailing newlines are ignored. Must have at least 8 fixed fields
                (through INFO)
            sample_names: a list of sample name strings; these should match
                the VCF header column
        Returns:
            A mutable VcfRecord.
        """
        vcf_fields = vcf_line.rstrip("\r\n").split("\t")
        chrom, pos, rid, ref, alt, qual, rfilter, info \
                = vcf_fields[0:8]
        sample_fields = []
        sample_tag_values = {}
        if len(vcf_fields) > 9:
            rformat = vcf_fields[8]
            sample_fields = vcf_fields[9:]
            sample_tag_values = VcfRecord._sample_tag_values(sample_names,
                                                             rformat,
                                                             sample_fields)
        return VcfRecord(chrom, pos, ref, alt,
                         rid, qual, rfilter, info,
                         sample_tag_values)

    @classmethod
    def _sample_tag_values(cls, sample_names, rformat, sample_fields):
        """Creates a sample dict of tag-value dicts for a single variant record.

        Args:
            sample_names: list of sample name strings.
            rformat: record format string (from VCF record).
            sample_fields: list of strings where each string is the ';'
                seperated format values for an individual sample.

        Returns:
            An dict of samples, where each key is a sample and each value
                is an dict of format-values. See attribute below for example.
                Will return '.' if no values for sampe field.
        """
        sample_tag_values = OrderedDict()
        tag_names = VcfRecord._format_list(rformat)
        for i, sample_field in enumerate(sample_fields):
            tag_values = sample_field.split(":") if sample_field else "."
            sample_tag_values[sample_names[i]] = OrderedDict(zip(tag_names,
                                                                 tag_values))
        return sample_tag_values

    @classmethod
    def _format_list(cls, rformat):
        if rformat and rformat != ".":
            return rformat.split(":")
        else:
            return []

    @classmethod
    def _str_as_int(cls, string):
        if "chr" in string:
            string = string.replace("chr", "")
        try:
            return int(string)
        except ValueError:
            return sys.maxsize
#TODO: (cgates): Could we make filter an OrderedSet
#TODO: (cgates) adjust info field to be stored as dict only instead of string
#pylint: disable=too-many-arguments
    def __init__(self, chrom, pos, ref, alt,
                 vcf_id=".", qual=".", vcf_filter=".", info=".",
                 sample_tag_values=None):
        """Builds a mutable VcfRecord from constituent VCF fields.

        If building from a VCF record line, see VcfRecord.parse_record.
        """
        self.chrom = chrom
        self.pos = pos
        self.vcf_id = vcf_id
        self.ref = ref
        self.alt = alt
        self.qual = qual
        self.filter = vcf_filter
        self.info = info
        self.info_dict = self._init_info_dict()

        if sample_tag_values is None:
            self.sample_tag_values = OrderedDict()
        else:
            self.sample_tag_values = sample_tag_values
        self._key = self._build_key()

    def _build_key(self):
        return (VcfRecord._str_as_int(self.chrom),
                self.chrom,
                VcfRecord._str_as_int(self.pos),
                self.ref,
                self.alt)

    @property
    def format_tags(self):
        """Returns set of format tags."""
        tags = VcfRecord._EMPTY_SET
        if self.sample_tag_values:
            first_sample = list(self.sample_tag_values.keys())[0]
            tags = set(self.sample_tag_values[first_sample].keys())
        return tags

    def _init_info_dict(self):
        info_dict = OrderedDict()
        if self.info and self.info != ".":
            info_list = self.info.split(";")
            for key_value in info_list:
                if "=" in key_value:
                    key, value = key_value.split("=")
                    info_dict[key] = value
                else:
                    info_dict[key_value] = key_value
        return info_dict

    def add_info_field(self, field):
        """Adds new info field (flag or key=value pair).

        Args:
            field: String flag (e.g. "SOMATIC") or key-value ("NEW_DP=42")

        Raises:
            KeyError: if info field already exists
        """
        if field in self.info_dict:
            msg = "New info field [{}] already exists.".format(field)
            raise KeyError(msg)

        if "=" in field:
            key, value = field.split("=")
            self.info_dict[key] = value
        else:
            self.info_dict[field] = field

        self._join_info_fields()

    #TODO:(cgates): Remove info; all external calls should reference info_dict
    def _join_info_fields(self):
        """Updates info attribute from info dict."""
        if self.info_dict:
            info_fields = []
            if len(self.info_dict) > 1:
                self.info_dict.pop(".", None)
            for field, value in self.info_dict.items():
                if field == value:
                    info_fields.append(value)
                else:
                    info_fields.append("=".join([field, value]))
            self.info = ";".join(info_fields)
        else:
            self.info = "."

    #TODO cgates: move this to merge
    def get_empty_record(self):
        return VcfRecord(chrom=self.chrom,
                         pos=self.pos,
                         ref=self.ref,
                         alt=self.alt)

    def _format_field(self):
        """Returns string representation of format field."""
        format_field = "."
        if self.sample_tag_values:
            first_sample = list(self.sample_tag_values.keys())[0]
            tag_names = self.sample_tag_values[first_sample].keys()
            if tag_names:
                format_field = ":".join(tag_names)
        return format_field

    def _sample_field(self, sample):
        """Returns string representation of sample-format values.

        Raises:
            KeyError: if requested sample is not defined.
        """
        tag_values = self.sample_tag_values[sample].values()
        if tag_values:
            return ":".join(tag_values)
        else:
            return "."

    def text(self):
        "Returns tab-delimited, newline terminated string of VcfRecord."
        stringifier = [self.chrom, self.pos, self.vcf_id, self.ref, self.alt,
                       self.qual, self.filter, self.info,
                       self._format_field()]

        for sample in self.sample_tag_values:
            stringifier.append(self._sample_field(sample))

        return "\t".join(stringifier) + "\n"

    def _samples_match(self, new_sample_values):
        return set(new_sample_values.keys()) == \
                set(self.sample_tag_values.keys())

    def add_sample_tag_value(self, tag_name, new_sample_values):
        """Appends a new format tag-value for all samples.

        Args:
            tag_name: string tag name; must not already exist
            new_sample

        Raises:
            KeyError: if tag_name to be added already exists
        """
        if tag_name in self.format_tags:
            msg = "New format value [{}] already exists.".format(tag_name)
            raise KeyError(msg)

        if not self._samples_match(new_sample_values):
            raise KeyError("Sample name values must match "
                           "existing sample names")
        for sample in self.sample_tag_values.keys():
            value = str(new_sample_values[sample])
            self.sample_tag_values[sample][tag_name] = value

    def add_or_replace_filter(self, new_filter):
        """Replaces null or blank filter or adds filter to existing list."""
        if self.filter.lower() in self._FILTERS_TO_REPLACE:
            self.filter = new_filter
        elif new_filter not in self.filter.split(";"):
            self.filter = ";".join([self.filter,
                                    new_filter])

    def __eq__(self, other):
        return isinstance(other, VcfRecord) and self._key == other._key

    def __hash__(self):
        return hash(self._key)

    def __lt__(self, other):
        return self._key < other._key

#TODO cgates: add context management to open/close
class FileWriter(object):
    """Trivial wrapper around os file to expedite testing."""
    def __init__(self, output_filepath):
        self.output_filepath = output_filepath
        self._file_writer = None

    @property
    def file_name(self):
        return os.path.basename(self.output_filepath)

    def open(self):
        self._file_writer = open(self.output_filepath, "w")

    def write(self, text):
        return self._file_writer.write(text)

    def close(self):
        self._file_writer.close()

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
                and self.__dict__ == other.__dict__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.output_filepath)

class FileReader(object):
    """Trivial wrapper around os file to expedite testing/natural sorting."""

    def __init__(self, input_filepath):
        self.input_filepath = input_filepath
        self.file_name = os.path.basename(input_filepath)
        self._file_reader = None

    def open(self):
        self._file_reader = open(self.input_filepath, "r")

    def read_lines(self):
        for line in self._file_reader:
            yield line

    def close(self):
        self._file_reader.close()

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
                and self.__dict__ == other.__dict__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.input_filepath)

    def __lt__(self, other):
        key = natsort.natsort_keygen()
        return key(self.file_name) < key(other.file_name)


from registrasion.models import commerce
from registrasion.models import inventory

from django.db.models import Case
from django.db.models import F, Q
from django.db.models import Sum
from django.db.models import When
from django.db.models import Value

from .batch import BatchController

from operator import attrgetter

class AllProducts(object):
    pass

class CategoryController(object):

    def __init__(self, category):
        self.category = category

    @classmethod
    def available_categories(cls, user, products=AllProducts):
        ''' Returns the categories available to the user. Specify `products` if
        you want to restrict to just the categories that hold the specified
        products, otherwise it'll do all. '''

        # STOPGAP -- this needs to be elsewhere tbqh
        from .product import ProductController

        if products is AllProducts:
            products = inventory.Product.objects.all().select_related(
                "category",
            )

        available = ProductController.available_products(
            user,
            products=products,
        )

        return sorted(set(i.category for i in available), key=attrgetter("order"))

    @classmethod
    @BatchController.memoise
    def user_remainders(cls, user):
        '''

        Return:
            Mapping[int->int]: A dictionary that maps the category ID to the
            user's remainder for that category.

        '''

        categories = inventory.Category.objects.all()

        cart_filter = (
            Q(product__productitem__cart__user=user) &
            Q(product__productitem__cart__status=commerce.Cart.STATUS_PAID)
        )

        quantity = When(
            cart_filter,
            then='product__productitem__quantity'
        )

        quantity_or_zero = Case(
            quantity,
            default=Value(0),
        )

        remainder = Case(
            When(limit_per_user=None, then=Value(99999999)),
            default=F('limit_per_user') - Sum(quantity_or_zero),
        )

        categories = categories.annotate(remainder=remainder)

        return dict((cat.id, cat.remainder) for cat in categories)


from .controllers.product import ProductController
from .models import commerce
from .models import inventory

from django import forms
from django.db.models import Q

class ApplyCreditNoteForm(forms.Form):

    required_css_class = 'label-required'

    def __init__(self, user, *a, **k):
        ''' User: The user whose invoices should be made available as
        choices. '''
        self.user = user
        super(ApplyCreditNoteForm, self).__init__(*a, **k)

        self.fields["invoice"].choices = self._unpaid_invoices

    def _unpaid_invoices(self):
        invoices = commerce.Invoice.objects.filter(
            status=commerce.Invoice.STATUS_UNPAID,
        ).select_related("user")

        invoices_annotated = [invoice.__dict__ for invoice in invoices]
        users = dict((inv.user.id, inv.user) for inv in invoices)
        for invoice in invoices_annotated:
            invoice.update({
                "user_id": users[invoice["user_id"]].id,
                "user_email": users[invoice["user_id"]].email,
            })

        key = lambda inv: (0 - (inv["user_id"] == self.user.id), inv["id"])  # noqa
        invoices_annotated.sort(key=key)

        template = (
            'Invoice %(id)d - user: %(user_email)s (%(user_id)d) '
            '-  $%(value)d'
        )
        return [
            (invoice["id"], template % invoice)
            for invoice in invoices_annotated
        ]

    invoice = forms.ChoiceField(
        required=True,
    )
    verify = forms.BooleanField(
        required=True,
        help_text="Have you verified that this is the correct invoice?",
    )

class CancellationFeeForm(forms.Form):

    required_css_class = 'label-required'

    percentage = forms.DecimalField(
        required=True,
        min_value=0,
        max_value=100,
    )

class ManualCreditNoteRefundForm(forms.ModelForm):

    required_css_class = 'label-required'

    class Meta:
        model = commerce.ManualCreditNoteRefund
        fields = ["reference"]

class ManualPaymentForm(forms.ModelForm):

    required_css_class = 'label-required'

    class Meta:
        model = commerce.ManualPayment
        fields = ["reference", "amount"]

# Products forms -- none of these have any fields: they are to be subclassed
# and the fields added as needs be. ProductsForm (the function) is responsible
# for the subclassing.

def ProductsForm(category, products):
    ''' Produces an appropriate _ProductsForm subclass for the given render
    type. '''

    # Each Category.RENDER_TYPE value has a subclass here.
    cat = inventory.Category
    RENDER_TYPES = {
        cat.RENDER_TYPE_QUANTITY: _QuantityBoxProductsForm,
        cat.RENDER_TYPE_RADIO: _RadioButtonProductsForm,
        cat.RENDER_TYPE_ITEM_QUANTITY: _ItemQuantityProductsForm,
        cat.RENDER_TYPE_CHECKBOX: _CheckboxProductsForm,
    }

    # Produce a subclass of _ProductsForm which we can alter the base_fields on
    class ProductsForm(RENDER_TYPES[category.render_type]):
        pass

    products = list(products)
    products.sort(key=lambda prod: prod.order)

    ProductsForm.set_fields(category, products)

    if category.render_type == inventory.Category.RENDER_TYPE_ITEM_QUANTITY:
        ProductsForm = forms.formset_factory(
            ProductsForm,
            formset=_ItemQuantityProductsFormSet,
        )

    return ProductsForm

class _HasProductsFields(object):

    PRODUCT_PREFIX = "product_"

    ''' Base class for product entry forms. '''
    def __init__(self, *a, **k):
        if "product_quantities" in k:
            initial = self.initial_data(k["product_quantities"])
            k["initial"] = initial
            del k["product_quantities"]
        super(_HasProductsFields, self).__init__(*a, **k)

    @classmethod
    def field_name(cls, product):
        return cls.PRODUCT_PREFIX + ("%d" % product.id)

    @classmethod
    def set_fields(cls, category, products):
        ''' Sets the base_fields on this _ProductsForm to allow selecting
        from the provided products. '''
        pass

    @classmethod
    def initial_data(cls, product_quantites):
        ''' Prepares initial data for an instance of this form.
        product_quantities is a sequence of (product,quantity) tuples '''
        return {}

    def product_quantities(self):
        ''' Yields a sequence of (product, quantity) tuples from the
        cleaned form data. '''
        return iter([])

    def add_product_error(self, product, error):
        ''' Adds an error to the given product's field '''

        ''' if product in field_names:
            field = field_names[product]
        elif isinstance(product, inventory.Product):
            return
        else:
            field = None '''

        self.add_error(self.field_name(product), error)

class _ProductsForm(_HasProductsFields, forms.Form):

    required_css_class = 'label-required'

    pass

class _QuantityBoxProductsForm(_ProductsForm):
    ''' Products entry form that allows users to enter quantities
    of desired products. '''

    @classmethod
    def set_fields(cls, category, products):
        for product in products:
            if product.description:
                help_text = "$%d each -- %s" % (
                    product.price,
                    product.description,
                )
            else:
                help_text = "$%d each" % product.price

            field = forms.IntegerField(
                label=product.name,
                help_text=help_text,
                min_value=0,
                max_value=500,  # Issue #19. We should figure out real limit.
            )
            cls.base_fields[cls.field_name(product)] = field

    @classmethod
    def initial_data(cls, product_quantities):
        initial = {}
        for product, quantity in product_quantities:
            initial[cls.field_name(product)] = quantity

        return initial

    def product_quantities(self):
        for name, value in self.cleaned_data.items():
            if name.startswith(self.PRODUCT_PREFIX):
                product_id = int(name[len(self.PRODUCT_PREFIX):])
                yield (product_id, value)

class _RadioButtonProductsForm(_ProductsForm):
    ''' Products entry form that allows users to enter quantities
    of desired products. '''

    FIELD = "chosen_product"

    @classmethod
    def set_fields(cls, category, products):
        choices = []
        for product in products:
            choice_text = "%s -- $%d" % (product.name, product.price)
            choices.append((product.id, choice_text))

        if not category.required:
            choices.append((0, "No selection"))

        cls.base_fields[cls.FIELD] = forms.TypedChoiceField(
            label=category.name,
            widget=forms.RadioSelect,
            choices=choices,
            empty_value=0,
            coerce=int,
        )

    @classmethod
    def initial_data(cls, product_quantities):
        initial = {}

        for product, quantity in product_quantities:
            if quantity > 0:
                initial[cls.FIELD] = product.id
                break

        return initial

    def product_quantities(self):
        ours = self.cleaned_data[self.FIELD]
        choices = self.fields[self.FIELD].choices
        for choice_value, choice_display in choices:
            if choice_value == 0:
                continue
            yield (
                choice_value,
                1 if ours == choice_value else 0,
            )

    def add_product_error(self, product, error):
        self.add_error(self.FIELD, error)

class _CheckboxProductsForm(_ProductsForm):
    ''' Products entry form that allows users to say yes or no
    to desired products. Basically, it's a quantity form, but the quantity
    is either zero or one.'''

    @classmethod
    def set_fields(cls, category, products):
        for product in products:
            field = forms.BooleanField(
                label='%s -- %s' % (product.name, product.price),
                required=False,
            )
            cls.base_fields[cls.field_name(product)] = field

    @classmethod
    def initial_data(cls, product_quantities):
        initial = {}
        for product, quantity in product_quantities:
            initial[cls.field_name(product)] = bool(quantity)

        return initial

    def product_quantities(self):
        for name, value in self.cleaned_data.items():
            if name.startswith(self.PRODUCT_PREFIX):
                product_id = int(name[len(self.PRODUCT_PREFIX):])
                yield (product_id, int(value))

class _ItemQuantityProductsForm(_ProductsForm):
    ''' Products entry form that allows users to select a product type, and
     enter a quantity of that product. This version _only_ allows a single
     product type to be purchased. This form is usually used in concert with
     the _ItemQuantityProductsFormSet to allow selection of multiple
     products.'''

    CHOICE_FIELD = "choice"
    QUANTITY_FIELD = "quantity"

    @classmethod
    def set_fields(cls, category, products):
        choices = []

        if not category.required:
            choices.append((0, "---"))

        for product in products:
            choice_text = "%s -- $%d each" % (product.name, product.price)
            choices.append((product.id, choice_text))

        cls.base_fields[cls.CHOICE_FIELD] = forms.TypedChoiceField(
            label=category.name,
            widget=forms.Select,
            choices=choices,
            initial=0,
            empty_value=0,
            coerce=int,
        )

        cls.base_fields[cls.QUANTITY_FIELD] = forms.IntegerField(
            label="Quantity",  # TODO: internationalise
            min_value=0,
            max_value=500,  # Issue #19. We should figure out real limit.
        )

    @classmethod
    def initial_data(cls, product_quantities):
        initial = {}

        for product, quantity in product_quantities:
            if quantity > 0:
                initial[cls.CHOICE_FIELD] = product.id
                initial[cls.QUANTITY_FIELD] = quantity
                break

        return initial

    def product_quantities(self):
        our_choice = self.cleaned_data[self.CHOICE_FIELD]
        our_quantity = self.cleaned_data[self.QUANTITY_FIELD]
        choices = self.fields[self.CHOICE_FIELD].choices
        for choice_value, choice_display in choices:
            if choice_value == 0:
                continue
            yield (
                choice_value,
                our_quantity if our_choice == choice_value else 0,
            )

    def add_product_error(self, product, error):
        if self.CHOICE_FIELD not in self.cleaned_data:
            return

        if product.id == self.cleaned_data[self.CHOICE_FIELD]:
            self.add_error(self.CHOICE_FIELD, error)
            self.add_error(self.QUANTITY_FIELD, error)

class _ItemQuantityProductsFormSet(_HasProductsFields, forms.BaseFormSet):

    required_css_class = 'label-required'

    @classmethod
    def set_fields(cls, category, products):
        raise ValueError("set_fields must be called on the underlying Form")

    @classmethod
    def initial_data(cls, product_quantities):
        ''' Prepares initial data for an instance of this form.
        product_quantities is a sequence of (product,quantity) tuples '''

        f = [
            {
                _ItemQuantityProductsForm.CHOICE_FIELD: product.id,
                _ItemQuantityProductsForm.QUANTITY_FIELD: quantity,
            }
            for product, quantity in product_quantities
            if quantity > 0
        ]
        return f

    def product_quantities(self):
        ''' Yields a sequence of (product, quantity) tuples from the
        cleaned form data. '''

        products = set()
        # Track everything so that we can yield some zeroes
        all_products = set()

        for form in self:
            if form.empty_permitted and not form.cleaned_data:
                # This is the magical empty form at the end of the list.
                continue

            for product, quantity in form.product_quantities():
                all_products.add(product)
                if quantity == 0:
                    continue
                if product in products:
                    form.add_error(
                        _ItemQuantityProductsForm.CHOICE_FIELD,
                        "You may only choose each product type once.",
                    )
                    form.add_error(
                        _ItemQuantityProductsForm.QUANTITY_FIELD,
                        "You may only choose each product type once.",
                    )
                products.add(product)
                yield product, quantity

        for product in (all_products - products):
            yield product, 0

    def add_product_error(self, product, error):
        for form in self.forms:
            form.add_product_error(product, error)

    @property
    def errors(self):
        _errors = super(_ItemQuantityProductsFormSet, self).errors
        if False not in [not form.errors for form in self.forms]:
            return []
        else:
            return _errors

class VoucherForm(forms.Form):

    required_css_class = 'label-required'

    voucher = forms.CharField(
        label="Voucher code",
        help_text="If you have a voucher code, enter it here",
        required=False,
    )

def staff_products_form_factory(user):
    ''' Creates a StaffProductsForm that restricts the available products to
    those that are available to a user. '''

    products = inventory.Product.objects.all()
    products = ProductController.available_products(user, products=products)

    product_ids = [product.id for product in products]
    product_set = inventory.Product.objects.filter(id__in=product_ids)

    class StaffProductsForm(forms.Form):
        ''' Form for allowing staff to add an item to a user's cart. '''

        product = forms.ModelChoiceField(
            widget=forms.Select,
            queryset=product_set,
        )

        quantity = forms.IntegerField(
            min_value=0,
        )

    return StaffProductsForm

def staff_products_formset_factory(user):
    ''' Creates a formset of StaffProductsForm for the given user. '''
    form_type = staff_products_form_factory(user)
    return forms.formset_factory(form_type)

class InvoicesWithProductAndStatusForm(forms.Form):

    required_css_class = 'label-required'

    invoice = forms.ModelMultipleChoiceField(
        widget=forms.CheckboxSelectMultiple,
        queryset=commerce.Invoice.objects.all(),
    )

    def __init__(self, *a, **k):
        category = k.pop('category', None) or []
        product = k.pop('product', None) or []
        status = int(k.pop('status', None) or 0)

        category = [int(i) for i in category]
        product = [int(i) for i in product]

        super(InvoicesWithProductAndStatusForm, self).__init__(*a, **k)

        qs = commerce.Invoice.objects.filter(
            status=status or commerce.Invoice.STATUS_UNPAID,
        ).filter(
            Q(lineitem__product__category__in=category) |
            Q(lineitem__product__in=product)
        )

        # Uniqify
        qs = commerce.Invoice.objects.filter(
            id__in=qs,
        )

        qs = qs.select_related("user__attendee__attendeeprofilebase")
        qs = qs.order_by("id")

        self.fields['invoice'].queryset = qs
        # self.fields['invoice'].initial = [i.id for i in qs] # UNDO THIS LATER

class InvoiceEmailForm(InvoicesWithProductAndStatusForm):

    ACTION_PREVIEW = 1
    ACTION_SEND = 2

    ACTION_CHOICES = (
        (ACTION_PREVIEW, "Preview"),
        (ACTION_SEND, "Send emails"),
    )

    from_email = forms.CharField()
    subject = forms.CharField()
    body = forms.CharField(
        widget=forms.Textarea,
    )
    action = forms.TypedChoiceField(
        widget=forms.RadioSelect,
        coerce=int,
        choices=ACTION_CHOICES,
        initial=ACTION_PREVIEW,
    )


import contextlib
import functools

from django.contrib.auth.models import User

class BatchController(object):
    ''' Batches are sets of operations where certain queries for users may be
    repeated, but are also unlikely change within the boundaries of the batch.

    Batches are keyed per-user. You can mark the edge of the batch with the
    ``batch`` context manager. If you nest calls to ``batch``, only the
    outermost call will have the effect of ending the batch.

    Batches store results for functions wrapped with ``memoise``. These results
    for the user are flushed at the end of the batch.

    If a return for a memoised function has a callable attribute called
    ``end_batch``, that attribute will be called at the end of the batch.

    '''

    _user_caches = {}
    _NESTING_KEY = "nesting_count"

    @classmethod
    @contextlib.contextmanager
    def batch(cls, user):
        ''' Marks the entry point for a batch for the given user. '''

        cls._enter_batch_context(user)
        try:
            yield
        finally:
            # Make sure we clean up in case of errors.
            cls._exit_batch_context(user)

    @classmethod
    def _enter_batch_context(cls, user):
        if user not in cls._user_caches:
            cls._user_caches[user] = cls._new_cache()

        cache = cls._user_caches[user]
        cache[cls._NESTING_KEY] += 1

    @classmethod
    def _exit_batch_context(cls, user):
        cache = cls._user_caches[user]
        cache[cls._NESTING_KEY] -= 1

        if cache[cls._NESTING_KEY] == 0:
            cls._call_end_batch_methods(user)
            del cls._user_caches[user]

    @classmethod
    def _call_end_batch_methods(cls, user):
        cache = cls._user_caches[user]
        ended = set()
        while True:
            keys = set(cache.keys())
            if ended == keys:
                break
            keys_to_end = keys - ended
            for key in keys_to_end:
                item = cache[key]
                if hasattr(item, 'end_batch') and callable(item.end_batch):
                    item.end_batch()
            ended = ended | keys_to_end

    @classmethod
    def memoise(cls, func):
        ''' Decorator that stores the result of the stored function in the
        user's results cache until the batch completes. Keyword arguments are
        not yet supported.

        Arguments:
            func (callable(*a)): The function whose results we want
                to store. The positional arguments, ``a``, are used as cache
                keys.

        Returns:
            callable(*a): The memosing version of ``func``.

        '''

        @functools.wraps(func)
        def f(*a):

            for arg in a:
                if isinstance(arg, User):
                    user = arg
                    break
            else:
                raise ValueError("One position argument must be a User")

            func_key = (func, tuple(a))
            cache = cls.get_cache(user)

            if func_key not in cache:
                cache[func_key] = func(*a)

            return cache[func_key]

        return f

    @classmethod
    def get_cache(cls, user):
        if user not in cls._user_caches:
            # Return blank cache here, we'll just discard :)
            return cls._new_cache()

        return cls._user_caches[user]

    @classmethod
    def _new_cache(cls):
        ''' Returns a new cache dictionary. '''
        cache = {}
        cache[cls._NESTING_KEY] = 0
        return cache


from registrasion.models import conditions
from registrasion.models import inventory

from symposion.proposals import models as proposals_models

from django import forms

# Reporting forms.

def mix_form(*a):
    ''' Creates a new form class out of all the supplied forms '''

    bases = tuple(a)
    return type("MixForm", bases, {})

class DiscountForm(forms.Form):

    required_css_class = 'label-required'

    discount = forms.ModelMultipleChoiceField(
        queryset=conditions.DiscountBase.objects.all(),
        required=False,
    )

class ProductAndCategoryForm(forms.Form):

    required_css_class = 'label-required'

    product = forms.ModelMultipleChoiceField(
        queryset=inventory.Product.objects.select_related("category"),
        required=False,
    )
    category = forms.ModelMultipleChoiceField(
        queryset=inventory.Category.objects.all(),
        required=False,
    )

class UserIdForm(forms.Form):

    required_css_class = 'label-required'

    user = forms.IntegerField(
        label="User ID",
        required=False,
    )

class ProposalKindForm(forms.Form):

    required_css_class = 'label-required'

    kind = forms.ModelMultipleChoiceField(
        queryset=proposals_models.ProposalKind.objects.all(),
        required=False,
    )

class GroupByForm(forms.Form):

    required_css_class = 'label-required'

    GROUP_BY_CATEGORY = "category"
    GROUP_BY_PRODUCT = "product"

    choices = (
        (GROUP_BY_CATEGORY, "Category"),
        (GROUP_BY_PRODUCT, "Product"),
    )

    group_by = forms.ChoiceField(
        label="Group by",
        choices=choices,
        required=False,
    )

def model_fields_form_factory(model):
    ''' Creates a form for specifying fields from a model to display. '''

    fields = model._meta.get_fields()

    choices = []
    for field in fields:
        if hasattr(field, "verbose_name"):
            choices.append((field.name, field.verbose_name))

    class ModelFieldsForm(forms.Form):
        fields = forms.MultipleChoiceField(
            choices=choices,
            required=False,
        )

    return ModelFieldsForm


''' NEEDS TESTS '''

import operator
from functools import reduce

from registrasion.models import commerce
from registrasion.models import inventory

from collections import Iterable
from collections import namedtuple
from django.db.models import Case
from django.db.models import Q
from django.db.models import Sum
from django.db.models import When
from django.db.models import Value

_ProductAndQuantity = namedtuple("ProductAndQuantity", ["product", "quantity"])

class ProductAndQuantity(_ProductAndQuantity):
    ''' Class that holds a product and a quantity.

    Attributes:
        product (models.inventory.Product)

        quantity (int)

    '''
    pass

class ItemController(object):

    def __init__(self, user):
        self.user = user

    def _items(self, cart_status, category=None):
        ''' Aggregates the items that this user has purchased.

        Arguments:
            cart_status (int or Iterable(int)): etc
            category (Optional[models.inventory.Category]): the category
                of items to restrict to.

        Returns:
            [ProductAndQuantity, ...]: A list of product-quantity pairs,
                aggregating like products from across multiple invoices.

        '''

        if not isinstance(cart_status, Iterable):
            cart_status = [cart_status]

        status_query = (
            Q(productitem__cart__status=status) for status in cart_status
        )

        in_cart = Q(productitem__cart__user=self.user)
        in_cart = in_cart & reduce(operator.__or__, status_query)

        quantities_in_cart = When(
            in_cart,
            then="productitem__quantity",
        )

        quantities_or_zero = Case(
            quantities_in_cart,
            default=Value(0),
        )

        products = inventory.Product.objects

        if category:
            products = products.filter(category=category)

        products = products.select_related("category")
        products = products.annotate(quantity=Sum(quantities_or_zero))
        products = products.filter(quantity__gt=0)

        out = []
        for prod in products:
            out.append(ProductAndQuantity(prod, prod.quantity))
        return out

    def items_pending_or_purchased(self):
        ''' Returns the items that this user has purchased or has pending. '''
        status = [commerce.Cart.STATUS_PAID, commerce.Cart.STATUS_ACTIVE]
        return self._items(status)

    def items_purchased(self, category=None):
        ''' Aggregates the items that this user has purchased.

        Arguments:
            category (Optional[models.inventory.Category]): the category
                of items to restrict to.

        Returns:
            [ProductAndQuantity, ...]: A list of product-quantity pairs,
                aggregating like products from across multiple invoices.

        '''
        return self._items(commerce.Cart.STATUS_PAID, category=category)

    def items_pending(self):
        ''' Gets all of the items that the user has reserved, but has not yet
        paid for.

        Returns:
            [ProductAndQuantity, ...]: A list of product-quantity pairs for the
                items that the user has not yet paid for.

        '''

        return self._items(commerce.Cart.STATUS_ACTIVE)

    def items_released(self):
        ''' Gets all of the items that the user previously paid for, but has
        since refunded.

        Returns:
            [ProductAndQuantity, ...]: A list of product-quantity pairs for the
                items that the user has not yet paid for.

        '''

        return self._items(commerce.Cart.STATUS_RELEASED)


import os

from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string
from django.utils.html import strip_tags

from django.contrib.sites.models import Site

class Sender(object):
    ''' Class for sending e-mails under a templete prefix. '''

    def __init__(self, template_prefix):
        self.template_prefix = template_prefix

    def send_email(self, to, kind, **kwargs):
        ''' Sends an e-mail to the given address.

        to: The address
        kind: the ID for an e-mail kind; it should point to a subdirectory of
            self.template_prefix containing subject.txt and message.html, which
            are django templates for the subject and HTML message respectively.

        context: a context for rendering the e-mail.

        '''

        return __send_email__(self.template_prefix, to, kind, **kwargs)

send_email = Sender("registrasion/emails").send_email

def __send_email__(template_prefix, to, kind, **kwargs):

    current_site = Site.objects.get_current()

    ctx = {
        "current_site": current_site,
        "STATIC_URL": settings.STATIC_URL,
    }
    ctx.update(kwargs.get("context", {}))
    subject_template = os.path.join(template_prefix, "%s/subject.txt" % kind)
    message_template = os.path.join(template_prefix, "%s/message.html" % kind)
    subject = "[%s] %s" % (
        current_site.name,
        render_to_string(subject_template, ctx).strip()
    )

    message_html = render_to_string(message_template, ctx)
    message_plaintext = strip_tags(message_html)

    from_email = settings.DEFAULT_FROM_EMAIL

    try:
        bcc_email = settings.ENVELOPE_BCC_LIST
    except AttributeError:
        bcc_email = None

    email = EmailMultiAlternatives(
        subject,
        message_plaintext,
        from_email,
        to,
        bcc=bcc_email,
    )
    email.attach_alternative(message_html, "text/html")
    email.send()


import pyosm.model as model
import datetime
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2
try:
    import StringIO
except ImportError:
    from io import StringIO
import gzip
import time
import os.path
from lxml import etree

def isoToDatetime(s):
    """Parse a ISO8601-formatted string to a Python datetime."""
    if s is None:
        return s
    else:
        return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ")

def noteTimeToDatetime(s):
    """Parse a datetime out of the Notes RSS feed."""
    if s is None:
        return s
    else:
        # 2013-05-01 08:10:42 UTC
        return datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S UTC")

def maybeInt(s):
    return int(s) if s is not None else s

def maybeFloat(s):
    return float(s) if s is not None else s

def maybeBool(s):
    return s == 'true' if s is not None else s

def readState(state_file, sep='='):
    state = {}

    for line in state_file:
        if line.startswith('---'):
            continue
        if line[0] == '#':
            continue
        (k, v) = line.split(sep)
        state[k] = v.strip().replace("\\:", ":")

    return state

def iter_changeset_stream(start_sqn=None, base_url='https://planet.openstreetmap.org/replication/changesets', expected_interval=60, parse_timestamps=True, state_dir=None):
    """Start processing an OSM changeset stream and yield one (action, primitive) tuple
    at a time to the caller."""

    # This is a lot like the other osm_stream except there's no
    # state file for each of the diffs, so just push ahead until
    # we run into a 404.

    # If the user specifies a state_dir, read the state from the statefile there
    if state_dir:
        if not os.path.exists(state_dir):
            raise Exception('Specified state_dir "%s" doesn\'t exist.' % state_dir)

        if os.path.exists('%s/state.yaml' % state_dir):
            with open('%s/state.yaml' % state_dir) as f:
                state = readState(f, ': ')
                start_sqn = state['sequence']

    # If no start_sqn, assume to start from the most recent changeset file
    if not start_sqn:
        u = urllib2.urlopen('%s/state.yaml' % base_url)
        state = readState(u, ': ')
        sequenceNumber = int(state['sequence'])
    else:
        sequenceNumber = int(start_sqn)

    interval_fudge = 0.0
    while True:
        sqnStr = str(sequenceNumber).zfill(9)
        url = '%s/%s/%s/%s.osm.gz' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9])

        delay = 1.0
        while True:
            try:
                content = urllib2.urlopen(url)
                content = StringIO.StringIO(content.read())
                gzipper = gzip.GzipFile(fileobj=content)
                interval_fudge -= (interval_fudge / 2.0)
                break
            except urllib2.HTTPError as e:
                if e.code == 404:
                    time.sleep(delay)
                    delay = min(delay * 2, 13)
                    interval_fudge += delay

        obj = None
        for event, elem in etree.iterparse(gzipper, events=('start', 'end')):
            if event == 'start':
                if elem.tag == 'changeset':
                    obj = model.Changeset(
                        int(elem.attrib['id']),
                        isoToDatetime(elem.attrib.get('created_at')) if parse_timestamps else elem.attrib.get('created_at'),
                        isoToDatetime(elem.attrib.get('closed_at')) if parse_timestamps else elem.attrib.get('closed_at'),
                        maybeBool(elem.attrib['open']),
                        maybeFloat(elem.get('min_lat')),
                        maybeFloat(elem.get('max_lat')),
                        maybeFloat(elem.get('min_lon')),
                        maybeFloat(elem.get('max_lon')),
                        elem.attrib.get('user'),
                        maybeInt(elem.attrib.get('uid')),
                        []
                    )
                elif elem.tag == 'tag':
                    obj.tags.append(
                        model.Tag(
                            elem.attrib['k'],
                            elem.attrib['v']
                        )
                    )
            elif event == 'end':
                if elem.tag == 'changeset':
                    yield obj
                    obj = None

        yield model.Finished(sequenceNumber, None)

        sequenceNumber += 1

        if state_dir:
            with open('%s/state.yaml' % state_dir, 'w') as f:
                f.write('sequence: %d' % sequenceNumber)

def iter_osm_change_file(f, parse_timestamps=True):
    action = None
    obj = None
    for event, elem in etree.iterparse(f, events=('start', 'end')):
        if event == 'start':
            if elem.tag == 'node':
                obj = model.Node(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    maybeFloat(elem.attrib.get('lat')),
                    maybeFloat(elem.attrib.get('lon')),
                    []
                )
            elif elem.tag == 'way':
                obj = model.Way(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []
                )
            elif elem.tag == 'tag':
                obj.tags.append(
                    model.Tag(
                        elem.attrib['k'],
                        elem.attrib['v']
                    )
                )
            elif elem.tag == 'nd':
                obj.nds.append(int(elem.attrib['ref']))
            elif elem.tag == 'relation':
                obj = model.Relation(
                    int(elem.attrib['id']),
                    int(elem.attrib['version']),
                    int(elem.attrib['changeset']),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []
                )
            elif elem.tag == 'member':
                obj.members.append(
                    model.Member(
                        elem.attrib['type'],
                        int(elem.attrib['ref']),
                        elem.attrib['role']
                    )
                )
            elif elem.tag in ('create', 'modify', 'delete'):
                action = elem.tag
        elif event == 'end':
            if elem.tag == 'node':
                yield (action, obj)
                obj = None
            elif elem.tag == 'way':
                yield (action, obj)
                obj = None
            elif elem.tag == 'relation':
                yield (action, obj)
                obj = None
            elif elem.tag in ('create', 'modify', 'delete'):
                action = None

        elem.clear()
        while elem.getprevious() is not None:
            del elem.getparent()[0]

def iter_osm_stream(start_sqn=None, base_url='https://planet.openstreetmap.org/replication/minute', expected_interval=60, parse_timestamps=True, state_dir=None):
    """Start processing an OSM diff stream and yield one changeset at a time to
    the caller."""

    # If the user specifies a state_dir, read the state from the statefile there
    if state_dir:
        if not os.path.exists(state_dir):
            raise Exception('Specified state_dir "%s" doesn\'t exist.' % state_dir)

        if os.path.exists('%s/state.txt' % state_dir):
            with open('%s/state.txt' % state_dir) as f:
                state = readState(f)
                start_sqn = state['sequenceNumber']

    # If no start_sqn, assume to start from the most recent diff
    if not start_sqn:
        u = urllib2.urlopen('%s/state.txt' % base_url)
        state = readState(u)
    else:
        sqnStr = str(start_sqn).zfill(9)
        u = urllib2.urlopen('%s/%s/%s/%s.state.txt' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9]))
        state = readState(u)

    interval_fudge = 0.0

    while True:
        sqnStr = state['sequenceNumber'].zfill(9)
        url = '%s/%s/%s/%s.osc.gz' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9])
        content = urllib2.urlopen(url)
        content = StringIO.StringIO(content.read())
        gzipper = gzip.GzipFile(fileobj=content)

        for a in iter_osm_change_file(gzipper, parse_timestamps):
            yield a

        # After parsing the OSC, check to see how much time is remaining
        stateTs = datetime.datetime.strptime(state['timestamp'], "%Y-%m-%dT%H:%M:%SZ")
        yield (None, model.Finished(state['sequenceNumber'], stateTs))

        nextTs = stateTs + datetime.timedelta(seconds=expected_interval + interval_fudge)
        if datetime.datetime.utcnow() < nextTs:
            timeToSleep = (nextTs - datetime.datetime.utcnow()).total_seconds()
        else:
            timeToSleep = 0.0
        time.sleep(timeToSleep)

        # Then try to fetch the next state file
        sqnStr = str(int(state['sequenceNumber']) + 1).zfill(9)
        url = '%s/%s/%s/%s.state.txt' % (base_url, sqnStr[0:3], sqnStr[3:6], sqnStr[6:9])
        delay = 1.0
        while True:
            try:
                u = urllib2.urlopen(url)
                interval_fudge -= (interval_fudge / 2.0)
                break
            except urllib2.HTTPError as e:
                if e.code == 404:
                    time.sleep(delay)
                    delay = min(delay * 2, 13)
                    interval_fudge += delay

        if state_dir:
            with open('%s/state.txt' % state_dir, 'w') as f:
                f.write(u.read())
            with open('%s/state.txt' % state_dir, 'r') as f:
                state = readState(f)
        else:
            state = readState(u)

def iter_osm_file(f, parse_timestamps=True):
    """Parse a file-like containing OSM XML and yield one OSM primitive at a time
    to the caller."""

    obj = None
    for event, elem in etree.iterparse(f, events=('start', 'end')):
        if event == 'start':
            if elem.tag == 'node':
                obj = model.Node(
                    int(elem.attrib['id']),
                    maybeInt(elem.get('version')),
                    maybeInt(elem.get('changeset')),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    maybeFloat(elem.get('lat')),
                    maybeFloat(elem.get('lon')),
                    []
                )
            elif elem.tag == 'way':
                obj = model.Way(
                    int(elem.attrib['id']),
                    maybeInt(elem.get('version')),
                    maybeInt(elem.get('changeset')),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []
                )
            elif elem.tag == 'tag':
                obj.tags.append(
                    model.Tag(
                        elem.attrib['k'],
                        elem.attrib['v']
                    )
                )
            elif elem.tag == 'nd':
                obj.nds.append(int(elem.attrib['ref']))
            elif elem.tag == 'relation':
                obj = model.Relation(
                    int(elem.attrib['id']),
                    maybeInt(elem.get('version')),
                    maybeInt(elem.get('changeset')),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    maybeBool(elem.attrib.get('visible')),
                    isoToDatetime(elem.attrib.get('timestamp')) if parse_timestamps else elem.attrib.get('timestamp'),
                    [],
                    []
                )
            elif elem.tag == 'member':
                obj.members.append(
                    model.Member(
                        elem.attrib['type'],
                        int(elem.attrib['ref']),
                        elem.attrib['role']
                    )
                )
            elif elem.tag == 'changeset':
                obj = model.Changeset(
                    int(elem.attrib['id']),
                    isoToDatetime(elem.attrib.get('created_at')) if parse_timestamps else elem.attrib.get('created_at'),
                    isoToDatetime(elem.attrib.get('closed_at')) if parse_timestamps else elem.attrib.get('closed_at'),
                    maybeBool(elem.attrib['open']),
                    maybeFloat(elem.get('min_lat')),
                    maybeFloat(elem.get('max_lat')),
                    maybeFloat(elem.get('min_lon')),
                    maybeFloat(elem.get('max_lon')),
                    elem.attrib.get('user'),
                    maybeInt(elem.attrib.get('uid')),
                    []
                )
        elif event == 'end':
            if elem.tag == 'node':
                yield obj
                obj = None
            elif elem.tag == 'way':
                yield obj
                obj = None
            elif elem.tag == 'relation':
                yield obj
                obj = None
            elif elem.tag == 'changeset':
                yield obj
                obj = None

        elem.clear()
        while elem.getprevious() is not None:
            del elem.getparent()[0]

def parse_osm_file(f, parse_timestamps=True):
    """Parse a file-like containing OSM XML into memory and return an object with
    the nodes, ways, and relations it contains. """

    nodes = []
    ways = []
    relations = []

    for p in iter_osm_file(f, parse_timestamps):

        if type(p) == model.Node:
            nodes.append(p)
        elif type(p) == model.Way:
            ways.append(p)
        elif type(p) == model.Relation:
            relations.append(p)

    return (nodes, ways, relations)

def get_note(note_id, parse_timestamps=True):
    u = urllib2.urlopen('https://www.openstreetmap.org/api/0.6/notes/%d' % note_id)
    tree = etree.parse(u)
    note_elem = tree.xpath('/osm/note')[0]

    def parse_comment(comment_element):
        user_elem = comment_element.xpath('user')
        uid_elem = comment_element.xpath('uid')
        return model.Comment(
            created_at=noteTimeToDatetime(comment_element.xpath('date')[0].text) if parse_timestamps else comment_element.xpath('date')[0].text,
            user=user_elem[0].text if user_elem else None,
            uid=int(uid_elem[0].text) if uid_elem else None,
            action=comment_element.xpath('action')[0].text,
            text=comment_element.xpath('text')[0].text
        )

    closed_elem = note_elem.xpath('date_closed')
    if closed_elem:
        closed_at = noteTimeToDatetime(closed_elem[0].text) if parse_timestamps else closed_elem[0].text
    else:
        closed_at = None

    return model.Note(
        id=int(note_elem.xpath('id')[0].text),
        lat=float(note_elem.attrib['lat']),
        lon=float(note_elem.attrib['lon']),
        created_at=noteTimeToDatetime(note_elem.xpath('date_created')[0].text) if parse_timestamps else note_elem.xpath('date_created')[0].text,
        closed_at=closed_at,
        status=note_elem.xpath('status')[0].text,
        comments=[parse_comment(c) for c in note_elem.xpath('comments/comment')]
    )

def iter_osm_notes(feed_limit=25, interval=60, parse_timestamps=True):
    """ Parses the global OSM Notes feed and yields as much Note information as possible. """

    last_seen_guid = None
    while True:
        u = urllib2.urlopen('https://www.openstreetmap.org/api/0.6/notes/feed?limit=%d' % feed_limit)

        tree = etree.parse(u)

        new_notes = []
        for note_item in tree.xpath('/rss/channel/item'):
            title = note_item.xpath('title')[0].text

            if title.startswith('new note ('):
                action = 'create'
            elif title.startswith('new comment ('):
                action = 'comment'
            elif title.startswith('closed note ('):
                action = 'close'

            # Note that (at least for now) the link and guid are the same in the feed.
            guid = note_item.xpath('link')[0].text

            if last_seen_guid == guid:
                break
            elif last_seen_guid == None:
                # The first time through we want the first item to be the "last seen"
                # because the RSS feed is newest-to-oldest
                last_seen_guid = guid
            else:
                note_id = int(guid.split('/')[-1].split('#c')[0])
                new_notes.append((action, get_note(note_id, parse_timestamps)))

        # We yield the reversed list because we want to yield in change order
        # (i.e. "oldest to most current")
        for note in reversed(new_notes):
            yield note

        yield model.Finished(None, None)

        time.sleep(interval)