def parse_downloads(request):
    parsed = Downloads()

    limit = request.registry.settings.get("shavar.max_downloads_chunks",
                                          10000)

    for lineno, line in enumerate(request.body_file):
        line = line.strip()

        if not line or line.isspace():
            continue

        line = line.decode()
        # Did client provide max size preference?
        if line.startswith("s;"):
            if lineno != 0:
                return ParseError("Size request can only be the first line!")
            req_size = line.split(";", 1)[1]
            # Almost certainly redundant due to stripping the line above
            req_size = req_size.strip()
            try:
                req_size = int(req_size)
            except ValueError:
                raise ParseError("Invalid requested size")
            parsed.req_size = req_size
            continue

        if ";" not in line:
            raise ParseError("Bad downloads request: no semi-colon")

        lname, chunklist = line.split(";", 1)
        if not lname or '-' not in lname:
            raise ParseError("Invalid list name: \"%s\"" % lname)
        info = DownloadsListInfo(lname, limit=limit)

        chunks = chunklist.split(":")
        # Check for MAC
        if len(chunks) >= 1 and chunks[-1] == "mac":
            if request.GET.get('pver') == '3.0':
                raise ParseError('MAC not supported in protocol version 3')
            info.wants_mac = True
            chunks.pop(-1)
        # Client claims to have chunks for this list
        if not chunks or (len(chunks) == 1 and not chunks[0]):
            parsed.append(info)
            continue
        # Uneven number of chunks should only occur if 'mac' was specified
        if len(chunks) % 2 != 0:
            raise ParseError("Invalid LISTINFO for %s" % lname)

        while chunks:
            ctype = chunks.pop(0)
            if ctype not in ('a', 's'):
                raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" % (ctype,
                                                                      lname))

            list_of_chunks = chunks.pop(0)
            for chunk in list_of_chunks.split(','):
                try:
                    chunk = int(chunk)
                except ValueError:
                    if chunk.find('-'):
                        low, high = chunk.split('-', 1)
                        # FIXME should probably be stricter about testing for
                        #       pure integers only on the input
                        try:
                            low = int(low)
                            high = int(high)
                        except ValueError:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))
                        if low >= high:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))

                        info.add_range_claim(ctype, low, high)
                else:  # Resist temptation to indent!  It's a try/except/else
                    info.add_claim(ctype, chunk)
        parsed.append(info)
    return parsed def parse_gethash(request):
    parsed = []

    # Early check to be sure we have something within the limits of a
    # reasonably sized header.  Reasonable size defined as an arbitrary max
    # of 2**8 bytes and a minimum of 3("4:4", a single prefix).  256 is
    # probably waaaaaaaaaaay too large for a gethash request header.
    eoh = request.body.find(b'\n')
    if eoh < 3 or eoh >= 256:
        raise ParseError("Improbably small or large gethash header size: %d"
                         % eoh)

    body_file = request.body_file

    # determine size of individual prefixes and length of payload
    header = body_file.readline().decode()
    try:
        prefix_len, payload_len = [int(x) for x in header.split(':', 1)]
    except ValueError:
        raise ParseError('Invalid prefix or payload size: "%s"' % header)
    if ((payload_len % prefix_len != 0)
            or (payload_len < prefix_len)):
        raise ParseError("Payload length invalid: \"%d\"" % payload_len)

    prefix_total = payload_len // prefix_len
    prefixes_read = 0
    total_read = 0
    while prefixes_read < prefix_total:
        prefix = body_file.read(prefix_len)
        if len(prefix) < prefix_len:
            break
        total_read += len(prefix)
        prefixes_read += 1
        parsed.append(prefix)

    if prefixes_read != prefix_total:
        raise ParseError("Hash read mismatch: client claimed %d, read %d" %
                         (prefix_total, prefixes_read))

    if total_read != payload_len:
        raise ParseError("Mismatch on gethash parse: client: %d, actual: %d" %
                         (payload_len, total_read))

    return set(parsed)  # unique-ify def parse_file_source(handle):
    """
    Parses a chunk list formatted file
    """
    # We should almost certainly* find the end of the first newline within the
    # first 32 bytes of the file.  It consists of a colon delimited string
    # with the following members:
    #
    #  - type of chunk: 'a' or 's' == 1
    #  - chunk number:  assuming len(2**32) == max of 10
    #  - number of bytes in the hash prefix size: 4 bytes for shavar or
    #                                             32 digest256 == max of 2
    #  - length of the raw data following in octets: len(2**32) == max of 10
    #
    #  These total 23 plus 3 bytes for colons plus one byte for the newline
    #  bring the grand total for likely maximum length to 27 with a minimum
    #  of 8 bytes("1:1:4:1\n").
    #
    #  So 32 byte read should be more than sufficient.
    #
    # * If 64 bit ints get involved, there are other issues to address

    parsed = ChunkList()
    while True:
        blob = handle.read(32)

        # Consume any unnecessary newlines in front of chunks
        blob = blob.lstrip(b'\n')

        if not blob:
            break

        if len(blob) < 8:
            raise ParseError("Incomplete chunk file? Could only read %d "
                             "bytes of header." % len(blob))

        eol = blob.find(b'\n')
        if eol < 8:
            raise ParseError('Impossibly short chunk header: "%s"' % eol)
        header = get_header(blob, eol)

        if header.count(':') != 3:
            raise ParseError('Incorrect number of fields in chunk header: '
                             '"%s"' % header)

        add_sub, chunk_num, hash_len, read_len = header.split(':', 3)

        if len(add_sub) != 1:
            raise ParseError('Chunk type is too long: "%s"' % header)
        if add_sub not in ('a', 's'):
            raise ParseError('Invalid chunk type: "%s"' % header)

        try:
            chunk_num = int(chunk_num)
            hash_len = int(hash_len)
            read_len = int(read_len)
        except ValueError:
            raise ParseError('Non-integer chunk values: "%s"' % header)

        if read_len % hash_len != 0:
            raise ParseError('Chunk data length not a multiple of prefix '
                             'size: "%s"' % header)

        blob = blob[eol + 1:]
        blob += handle.read(read_len - len(blob))
        if blob is None or len(blob) < read_len:
            raise ParseError('Chunk data truncated for chunk %d' % chunk_num)

        hashes = []
        pos = 0
        while pos < read_len:
            hashes.append(blob[pos:pos + hash_len])
            pos += hash_len

        parsed.insert_chunk(Chunk(chunk_type=add_sub, number=chunk_num,
                                  hashes=hashes))

    return parsed def parse_dir_source(handle, exists_cb=os.path.exists, open_cb=open):
    """
    Expects a file alike object with the contents of a JSON formatted index
    file that has the following structure:

    {
        "name": "mozpub-tracking-digest256",
        "basedir": "mozpub-tracking-digest256",
        "chunks": {
            "1": {
                "path": "mozpub-tracking-digest256/1",
                "hashes(optional)": [ "", "" ],
                "prefixes(optional)": [ "", "" ]
            },
            "2": {
                "path": "mozpub-tracking-digest256/2",
                "hashes": [ "", "" ],
                "prefixes": [ "", "" ]
            }
        }
    }

    The basedir, hashes, and prefixes entries are optional.  The chunks to be
    served will be parsed with parse_file_source().  If hashes and prefixes are
    provided, they will be verified against the data provided in the given
    chunk file.
    """
    try:
        index = json.load(handle)
    except ValueError as e:
        raise ParseError("Could not parse index file: %s" % e)

    if 'name' not in index:
        raise ParseError("Incorrectly formatted index: missing list name")

    if 'chunks' not in index:
        raise ParseError("Incorrectly formatted index: missing chunks")

    if 'basedir' in index:
        basedir = posixpath.join(os.path.dirname(handle.name),
                                 index['basedir'])
    else:
        handle_name = handle.name
        if isinstance(handle_name, int):
            handle_name = str(handle_name)
        basedir = os.path.dirname(handle_name)

    parsed = ChunkList()
    int_key_chunks = {}
    for key in index['chunks'].keys():
        # A little massaging to make the data structure a little cleaner
        try:
            int_key_chunks[int(key)] = index['chunks'][key]
        except KeyError:
            raise ParseError("Some weird behaviour with the list of chunks "
                             "in \"%s\"" % handle.filename)
        chunk_file = posixpath.join(basedir, str(key))

        if not exists_cb(chunk_file):
            raise ParseError("Invalid chunk filename: \"%s\"" % chunk_file)

        with open_cb(chunk_file, 'rb') as f:
            chunk_list = parse_file_source(f)

        # Only one chunk per file
        if len(chunk_list) > 1:
            raise ParseError("More than one chunk in chunk file \"%s\""
                             % chunk_file)

        for chunk in itertools.chain(iter(chunk_list.adds.values()),
                                     iter(chunk_list.subs.values())):
            parsed.insert_chunk(chunk)
    index['chunks'] = int_key_chunks
    return parsed class Chunk(object):
    "Object for ease of interacting with parsed chunk data"

    def __init__(self, chunk_type='a', number=None, hashes=[], hash_size=32):
        if chunk_type not in ('a', 's'):
            raise ValueError('Invalid chunk type: "%s"' % chunk_type)

        if number is None:
            raise ValueError('Invalid chunk number: "%d"' % number)

        self.type = chunk_type
        self.number = number
        self.hashes = set(hashes)
        self.hash_len = hash_size
        self._prefix_cache = {}

    def __repr__(self):
        return "%s(chunk_type='%s', number=%d, hashes=%s, hash_size=%d)" \
            % (self.__class__.__name__, self.type, self.number, self.hashes,
               self.hash_len)

    def __eq__(self, other):
        if (type(self) != type(other)
                or self.type != other.type
                or self.number != other.number
                or self.hashes != other.hashes
                or self.hash_len != other.hash_len):
            return False
        return True

    def find_prefix(self, prefix):
        if prefix in self._prefix_cache:
            return True
        for hash_ in self.hashes:
            if hash_.startswith(prefix):
                self._prefix_cache[prefix] = True
                return True
        return False

    def get_hashes(self, prefix):
        hashes = []
        for hash_ in self.hashes:
            if hash_.startswith(prefix):
                hashes.append(hash_)
        return hashes class ChunkList(object):
    "Simplify interaction with server side lists of chunks"

    def __init__(self, add_chunks=[], sub_chunks=[]):
        self.adds = {}
        self.subs = {}
        for chunk in add_chunks:
            self.adds[chunk.number] = chunk
        for chunk in sub_chunks:
            self.subs[chunk.number] = chunk

    def __repr__(self):
        return "%s(add_chunks=%s, sub_chunks=%s)" \
            % (self.__class__.__name__,
               list(self.adds.values()),
               list(self.subs.values()))

    def __eq__(self, other):
        if (type(self) != type(other)
                or self.adds != other.adds
                or self.subs != other.subs):
            return False
        return True

    def __len__(self):
        return len(self.adds) + len(self.subs)

    def find_prefix(self, prefix):
        list_o_chunks = []
        for number, chunk in self.adds.items():
            if chunk.find_prefix(prefix):
                list_o_chunks.append(chunk)
        return list_o_chunks

    def insert_chunk(self, chunk):
        chunk_list = self.adds
        if chunk.type == 's':
            chunk_list = self.subs
        if chunk.number in chunk_list:
            raise ValueError("Duplicate chunk number: %d" % chunk.number)
        chunk_list[chunk.number] = chunk class Downloads(list):

    def __init__(self, req_size=0):
        if type(req_size) != int:
            raise TypeError('req_size not an integer: "%s"' % req_size)

        self.req_size = req_size
        super(Downloads, self).__init__()

    def __eq__(self, other):
        if (type(self) != type(other)
                or self.req_size != other.req_size
                or not super(Downloads, self).__eq__(other)):
            return False
        return True class DownloadsListInfo(object):

    def __init__(self, list_name, wants_mac=False, adds=[], subs=[],
                 limit=10 * 1000):
        self.name = list_name
        self.wants_mac = wants_mac
        self.limit = limit
        if len(adds) > self.limit:
            raise LimitExceededError("Number of add chunks(%d) exceeds limit:"
                                     " %d" % (len(adds), self.limit))
        self.adds = set(adds)
        if len(subs) > self.limit:
            raise LimitExceededError("Number of sub chunks(%d) exceeds limit:"
                                     " %d" % (len(subs), self.limit))
        self.subs = set(subs)

    def add_claim(self, typ, chunk_num):
        # We do bounds checking after the fact because sets only store unique
        # items and don't duplicate
        if typ == 's':
            self.subs.add(chunk_num)
            if len(self.subs) > self.limit:
                raise LimitExceededError("Number of sub chunks(%d) exceeds"
                                         " limit: %d" % (len(self.subs),
                                                         self.limit))
        else:
            self.adds.add(chunk_num)
            if len(self.adds) > self.limit:
                raise LimitExceededError("Number of add chunks(%d) exceeds"
                                         " limit: %d" % (len(self.adds),
                                                         self.limit))

    def add_range_claim(self, typ, low, high):
        for i in range(low, high + 1):
            self.add_claim(typ, i)

    def __eq__(self, other):
        if (type(self) != type(other)
                or self.name != other.name
                or self.wants_mac != other.wants_mac
                or self.adds != other.adds
                or self.subs != other.subs):
            return False
        return True

    def __repr__(self):
        return "%s('%s', wants_mac=%s, adds=%s, subs=%s)" \
            % (self.__class__.__name__, self.name, self.wants_mac, self.adds,
               self.subs) class LimitExceededError(Exception):
    """
    Raised when a /downloads request would exceed the limit of the number
    of chunks for a given request
    """ class DummyRequest(testing.DummyRequest):
class ShavarTestCase(TestCase):
    def body_file(self):
def dummy(body, path="/downloads", **kwargs):
def test_file(fname):
    def setUp(self):
    def get_configurator(self):
def chunkit(n, typ, *urls):
DELTA_RESULT = ChunkList(add_chunks=[chunkit(1, 'a',
                                             'https://www.mozilla.org/',
                                             'https://www.google.com/'),
                                     chunkit(2, 'a', 'https://github.com/',
                                             'http://www.python.org/'),
                                     chunkit(4, 'a',
                                             'http://www.haskell.org/',
                                             'https://www.mozilla.com/'),
                                     chunkit(5, 'a', 'http://www.erlang.org',
                                             'http://golang.org/')],
                         sub_chunks=[chunkit(3, 's',
                                             'https://github.com/'),
                                     chunkit(6, 's',
                                             'http://golang.org')])
import io
from shavar.exceptions import ParseError
from shavar.parse import (
    parse_downloads,
    parse_gethash,
    parse_file_source,
    parse_dir_source)
from shavar.types import (
    Chunk,
    ChunkList,
    Downloads,
    DownloadsListInfo,
    LimitExceededError)
from shavar.tests.base import (
    DELTA_RESULT,
    dummy,
    hashes,
    test_file,
    ShavarTestCase)
        # with multiple lists
        s = "googpub-phish-shavar;a:1-3,5:s:4-5\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)
        dli0.add_range_claim('a', 1, 3)
        dli0.add_claim('a', 5)
        dli0.add_range_claim('s', 4, 5)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)

        # with multiple lists, at least one empty
        # See https://github.com/mozilla-services/shavar/issues/56
        s = "googpub-phish-shavar;\n"
        s += "acme-white-shavar;a:1-7:s:1-2"
        p = parse_downloads(dummy(s))

        d = Downloads()
        dli0 = DownloadsListInfo("googpub-phish-shavar")
        d.append(dli0)

        dli1 = DownloadsListInfo("acme-white-shavar")
        d.append(dli1)
        dli1.add_range_claim('a', 1, 7)
        dli1.add_range_claim('s', 1, 2)
        self.assertEqual(p, d)

    def test_parse_download_errors(self):
        self.assertRaises(LimitExceededError, parse_downloads,
                          dummy("mozpub-track-digest256;a:1-20000"))

        self.assertRaises(LimitExceededError, parse_downloads,
                          dummy("mozpub-track-digest256;a:1-1002"))

        self.assertRaises(ParseError, parse_downloads,
                          dummy("mozpub-track-digest256"))

    def test_parse_gethash(self):
        h = b"4:32\n"
        d = (b"\xdd\x01J\xf5",
             b"\xedk8\xd9",
             b"\x13\x0e?F",
             b"o\x85\x0eF",
             b"\xd2\x1b\x95\x11",
             b"\x99\xd5:\x18",
             b"\xef)\xee\x93",
             b"AaN\xaf")
        s = b''
        s += h
        for i in d:
            s += i
        p = parse_gethash(dummy(s, path="/gethash"))
        self.assertEqual(p, set(d))
        # Make sure no repeats of issue #32 pop up: test with a single hash
        # prefix
        s = b"4:4\n\xdd\x01J\xf5"
        p = parse_gethash(dummy(s, path="/gethash"))
        self.assertEqual(p, set([b"\xdd\x01J\xf5"]))

    def test_parse_gethash_errors(self):
        # Too short
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("4:\n"))
        self.assertEqual(str(ecm.exception),
                         "Improbably small or large gethash header size: 2")
        # Too long
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("4:" + "1" * 256 + "\n"))
        self.assertEqual(str(ecm.exception),
                         "Improbably small or large gethash header size: 258")
        # Invalid sizes
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("steve:4\n"))
        self.assertEqual(str(ecm.exception),
                         'Invalid prefix or payload size: "steve:4\n"')
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("4:steve\n"))
        self.assertEqual(str(ecm.exception),
                         'Invalid prefix or payload size: "4:steve\n"')
        # Improper payload length
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("4:17\n"))
        self.assertEqual(str(ecm.exception),
                         'Payload length invalid: "17"')
        # Ditto but with a payload shorter than the prefix
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("8:4\n"))
        self.assertEqual(str(ecm.exception),
                         'Payload length invalid: "4"')
        # It seems some clients are hitting the gethash endpoint with a
        # request intended for the downloads endpoint
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("mozpub-track-digest256;a:1423242002"))
        self.assertEqual(str(ecm.exception),
                         "Improbably small or large gethash header size: -1")
        # See https://github.com/mozilla-services/shavar/issues/67
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy("1:10000000000\n"))
        self.assertEqual(str(ecm.exception),
                         "Hash read mismatch: client claimed 10000000000, "
                         "read 0")
        # Stated length of payload is longer than actual payload.  Only 7
        # bytes instead of 8 here.
        with self.assertRaises(ParseError) as ecm:
            parse_gethash(dummy(b"4:8\n\xdd\x01J\xf5\xedk8"))
        self.assertEqual(str(ecm.exception),
                         "Hash read mismatch: client claimed 2, read 1")

    def test_parse_file_source(self):
        d = b''.join([self.hm, self.hg])
        add = b"a:17:32:%d\n%s" % (len(d), d)
        sub = b"s:18:32:%d\n%s" % (len(d), d)


        adds = [Chunk(chunk_type='a', number=17, hashes=set([self.hg,