<filename>internal/llrp/generate_param_code.py
#
#  Copyright (C) 2020 Intel Corporation
#
#  SPDX-License-Identifier: Apache-2.0

# This python script reads a YAML file
# to generate Go code that unpacks binary LLRP messages
# so that they can be marshaled into JSON.
#
# Why bother doing it this way? In short, because other ways weren't good;
# it's much easier to express the LLRP message hierarchy as YAML than as Go code,
# and it's pretty easy to use that to generate parsers that are far less error prone
# than writing it by hand.
#
# At first blush, LLRP looks easy to parse,
# but when you dig in, there are a lot of rules and required context
# around how the next bytes should be interpreted.
# I tried writing it with interfaces and with reflection code,
# but it was way, way too easy to make a mistake, and I spent a ton of time
# debugging difficult to spot errors that could only be triggered at runtime.
#
# The best version came from 'hand-crafted' byte unmarshaling,
# but 1) it's still easy to mess up and 2) it would take too long to write.
#
# As I was building up a mental model around LLRP,
# I built a spreadsheet to reference the memory layouts, types, and rules.
# Looking at it, I realized it had all the need parsing info,
# so I converted to it the messages.yaml file and cleaned it up a bit,
# then wrote this code to read it in, statically check various assumptions,
# calculate some useful constraint checks from the message graph,
# and output the unmarshaling code.

try:
    import yaml
except ImportError:
    print('error: missing yaml; try "pip install pyyaml"')
    exit(1)

import dataclasses
import re
import sys
from contextlib import contextmanager, ExitStack
from dataclasses import dataclass
from itertools import groupby
from typing import Dict, List, Optional, Union, Any, Callable

YML = Dict[str, Any]  # just for type checking; otherwise does nothing


class Error(Exception):
    """Base class for errors generated by this script."""


class MissingParamError(Error):
    """Message or Parameter references an unknown Parameter."""

    def __init__(self, parent: 'Container', decl: 'ParamSpec', msg: str = None):
        if msg is None:
            msg = f'{parent.name} references unknown parameters {decl.param_name}'
        super(MissingParamError, self).__init__(msg)
        self.parent = parent
        self.decl = decl


class DefinitionError(Error):
    """Problem with the YAML definition."""

    def __init__(self, y: YML, msg=None):
        if msg is None:
            if 'name' in y:
                msg = f'failed at: {y["name"]}'
            elif 'type' in y:
                msg = f'failed at: {y["type"]}'
            else:
                msg = f'failed at: {y}'
        super(DefinitionError, self).__init__(msg)
        self.yml = y


class DuplicateDefError(DefinitionError):
    def __init__(self, kind: str, name: str, first: Any, second: Any, y: YML, msg=None):
        if msg is None:
            msg = f'duplicate {kind} definition for {name}:\n'
            f'> first instance: {first}\n'
            f'> this instance:  {second}\n'
        super(DuplicateDefError, self).__init__(y, msg)
        self.kind = kind
        self.name = name
        self.first = first
        self.second = second


class MissingPropError(DefinitionError):
    def __init__(self, missing: List[str], y: YML, msg=None):
        if msg is None:
            msg = f'missing required properties {missing}'
        super(MissingPropError, self).__init__(y, msg)


class GoWriter:
    """GoWriter outputs (mostly) formatted Go code.
    By default it writes to stdout, but you can set w to any File object
    (really, anything with a 'write' function)."""

    # map of basic types to byte lengths
    typeSizes = {
        'bool':   1,
        'int8':   1,
        'int16':  2,
        'int32':  4,
        'int64':  8,
        'uint8':  1,
        'uint16': 2,
        'uint32': 4,
        'uint64': 8,
    }

    PAREN = '()'
    BRACE = '{}'

    def __init__(self, w=None):
        import textwrap
        if w is None:
            self.w = sys.stdout
        else:
            self.w = w
        self.i = 0
        self.begin = True

        self.code = textwrap.TextWrapper(
            break_long_words=False,
            break_on_hyphens=False)

        self.buffered = None

    def noindent(self, text):
        self.w.write(text)
        self.w.write('\n')

    def buffer(self, text, pre='', auto_break=False):
        self.buffered = (text, pre, auto_break)

    def write_buffered(self):
        if self.buffered is None:
            return
        self.write(*self.buffered)

    def write(self, text, pre='', auto_break=False):
        self.buffered = None

        indent = self.i * '\t' + pre
        if not auto_break:
            for line in text.splitlines():
                self.w.write(indent + line.strip() + '\n')
            return

        requote = pre != '// '

        def write_paragraph(p):
            self.code.width = 70 - len(indent) if requote else 90 - len(indent)
            open_quote = False
            long_return = False
            for line in self.code.wrap(p):
                if long_return:
                    long_return = False
                    self.w.write(' ')
                else:
                    self.w.write(indent)

                if open_quote and requote:
                    self.w.write('"')

                self.w.write(line)

                if requote and line.count('"') & 1 == 1:
                    open_quote = not open_quote

                if open_quote:
                    self.w.write(' " +')

                if requote and line.strip() == 'return':
                    long_return = True
                else:
                    self.w.write('\n')

        paragraphs = text.split('\n')
        for p in paragraphs[:-1]:
            write_paragraph(p)
            self.w.write(pre + '\n')
        if paragraphs[-1].strip() != '':
            write_paragraph(paragraphs[-1])

    def comment(self, text, auto_break=True):
        self.write(text, pre='// ', auto_break=auto_break)

    @staticmethod
    def lower_camel(s: str) -> str:
        """Convert ABCCamelCaseString to abcCamelCaseString.
        Useful for creating unexported type names."""
        if len(s) <= 1:
            return s.lower()
        m = re.match(r'[A-Z]+', s)
        if not m or m.end() == len(s):
            return s.lower()

        if m.end() == 1:
            return s[0].lower() + s[1:]
        if s[m.end()].isalpha():
            return s[:m.end() - 1].lower() + s[m.end() - 1:]
        else:
            return s[:m.end()].lower() + s[m.end():]

    def indent(self):
        """Increase the indentation level for all subsequent output."""
        self.i += 1
        self.begin = True

    def dedent(self):
        """Decrease the indentation level fro all subsequent output."""
        self.i = max(self.i - 1, 0)
        self.begin = True

    @contextmanager
    def block(self, before_start: str = '', style=BRACE, after_end: str = '\n'):
        """Returns a context within which writes are indented inside braces,
        defined by the style. By default, this looks like:

            {
                <output written within context>
            }
            <next write goes here>

        Use like any other python context. E.g., this code:
            with writer.block('const', style='()'):
                for i in range(3):
                    w.write(f'myNum{i} = 1<<{i}')
        has this output:
            const (
                myNum0 = 1<<0
                myNum1 = 1<<1
                myNum2 = 1<<2
            )
        """
        if len(style) != 2:
            raise Error('style must have length 2')
        self.write(f'{before_start.strip()} {style[0]}'.strip())
        self.indent()
        yield
        self.dedent()
        self.write(f'{style[1]}{after_end}')

    @contextmanager
    def paren(self, before_start: str = ''):
        """Returns a context within which writes are indented inside parentheses."""
        with self.block(before_start, style=GoWriter.PAREN):
            yield

    @contextmanager
    def condition(self, cond: str):
        """Returns a context within which writes are indented inside an if (cond) block."""
        with self.block(before_start=f'if {cond}'):
            yield

    def ifelse(self, cond: Optional[str] = None):
        """Write the '} else {' or '} else if (condition) {' in an if...else block."""
        self.dedent()
        if cond:
            self.write(f'}} else if {cond} {{')
        else:
            self.write('} else {')
        self.indent()

    @contextmanager
    def switch(self, before_start: str = ''):
        """Returns a context for writing switch cases inside curly braces.
        See paren for more information."""
        with self.block(f'switch {before_start}'):
            yield

    @contextmanager
    def case(self, case: str):
        if case == 'default':
            self.write(f'default:')
        else:
            self.write(f'case {case}:')
        self.indent()
        yield
        self.dedent()

    def err_check(self, cond, params: Optional[List[str]] = None, ret: Optional[str] = 'err'):
        """Write 'if err := (cond); err != nil {\n return err\n }.
        Optionally, set other parameters and/or change the return value:
            if params[0], params[1], err := cond; err != nil {\n return ret\n}
        """
        p_list = ['err'] if not params else params + ['err']
        with self.block(f'if {", ".join(p_list)} := {cond}; err != nil'):
            if ret is not None:
                self.write('return ' + ret)

    def reterr(self, msg: Optional[str], params: Optional[List[str]] = None, wrap: bool = False):
        """Writes code to return an error, using 'return err' or one of
        New, Errorf, Wrap, or Wrapf if there are params and/or wrap is True."""
        if msg and (msg.count('%') != (len(params) if params else 0)):
            raise Error(f'got "{params}", but need {msg.count("%")} params for this error: "{msg}"')

        if not msg:
            assert params is None and wrap is False
            self.write('return err')
            return

        msg = msg.replace('\n', '" +\n"')
        if wrap and params:
            self.write(f'return errors.Wrapf(err, "{msg}", {", ".join(params)})', auto_break=True)
        elif wrap and not params:
            self.write(f'return errors.Wrap(err, "{msg}")')
        elif params:
            self.write(f'return errors.Errorf("{msg}", {", ".join(params)})', auto_break=True)
        else:
            self.write(f'return errors.New("{msg}")')


@dataclass
class DataType:
    """DataType backs field types, such as built-ins, flags, enumerations, and aliases."""
    storage: str  # underlying basic type (e.g., uint8)
    name: str = ''  # type name must be unique to the Go package
    prefix: str = ''  # when generating code, prefix is attached to enum/flag names
    kind: str = 'external'  # flag, enumeration, or alias, or external
    values: Optional[Dict[str, int]] = None  # maps name -> value, or implicitly name -> iota
    size: int = None  # storage byte size; for arrays, the byte size of a single element

    bits: int = 8  # number of bits in the final byte; usually should be 8, but occasionally not

    description: Optional[str] = None  # for outputting doc comments

    # used for testing; set to reasonable values if not specified in YAML
    min: Optional[int] = None
    max: Optional[int] = None

    @classmethod
    def from_yaml(cls, y: YML, types: Dict[str, 'DataType']) -> 'DataType':
        missing = [req for req in ('name', 'storage') if req not in y]
        if any(missing):
            raise MissingPropError(missing, y)

        if 'kind' not in y:
            y['kind'] = 'alias'

        try:
            return cls(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    def __post_init__(self):
        if self.name == '':
            self.name = self.storage

        if self.size is None:
            if self.storage.startswith('uint'):
                self.size = int(self.storage[4:]) // 8
            elif self.storage.startswith('int'):
                self.size = int(self.storage[3:]) // 8
            elif self.storage in ('bool', 'byte'):
                self.size = 1
            else:
                raise Error(f"the size of {self.name} ({self.storage}) isn't known")

        assert self.size > 0, self.name
        bts = 8 * self.size - 8 + self.bits
        if self.storage.startswith('int'):
            self.min = self.min if self.min is not None else -(1 << (bts - 1))
            self.max = self.max if self.max is not None else (1 << (bts - 1)) - 1
        else:
            self.min = self.min or 0
            self.max = self.max if self.max is not None else 1 << bts
        assert self.min is not None and self.max is not None, self.name

        if self.kind in ('alias', 'external'):
            return

        if self.kind not in ('flag', 'enum'):
            raise Error(f'unknown kind {self.kind} for type def {self.name}')

        if self.values is None:
            self.values = {}
            return self

        try:
            vals = {}
            for name, value in self.values.items():
                vals[self.prefix + name] = value
            self.values = vals
        except AttributeError:
            # allow the YAML value to use a list to map name -> position
            vals = {}
            for value, name in enumerate(self.values):
                vals[self.prefix + name] = value
            self.values = vals

        if len(self.values) > 1 << self.size * 8:
            raise Error(f'{self.name} has more items than can fit in {self.storage}')

        self.min = 0
        self.max = len(self.values)

        return self

    def write_type_def(self, w):
        if self.kind == 'external':
            return

        if self.description:
            w.comment(self.description)

        w.write(f'type {self.name}{"=" if self.is_alias() else " "}{self.storage}')
        if self.is_alias() or not any(self.values):
            return

        if len(self.values) == 1:
            name, value = list(self.values.items())[0]
            if self.kind == 'flag':
                w.write(f'const {name} = {self.name}(1 << {self.size * 8 - 1})')
            else:
                w.write(f'const {name} = {self.name}({value})')
            return

        if self.kind == 'enum':
            with w.paren('const'):
                for name, value in sorted(self.values.items(), key=lambda x: x[1]):
                    w.write(f'{name} = {self.name}({value})')
            w.write('')
            return

        assert self.kind == 'flag'
        flags = sorted(self.values.items(), key=lambda x: x[1])

        with w.paren('const'):
            w.write(f'{flags[0][0]} = {self.name}(1 << ({self.size * 8 - 1} - iota))')
            for name, _ in flags[1:]:
                w.write(f'{name}')

    def is_fixed_size(self) -> bool:
        """False for array types and True for all others."""
        return self.storage[0] != '['

    def is_alias(self) -> bool:
        return self.kind == 'alias'

    def is_enum(self) -> bool:
        return self.kind == 'enum'

    def is_external(self) -> bool:
        return self.kind == 'external'

    def can_copy(self) -> bool:
        return (self.storage in ('byte', 'uint8') and
                (self.is_alias() or self.name in ('byte', 'uint8')))

    def test_instance(self, i: int) -> str:
        """Returns valid Go code for an instance of the correct type,
        possibly using i as a seed.
        For a given positive value of i, this always returns the same value,
        but different values of i won't necessarily result in different values.
        """
        if self.name == 'string':
            return '"some arbitrary string"'
        if self.name == 'bool':
            return 'true' if i & 1 == 0 else 'false'

        assert self.max is not None and self.min is not None, self.name
        dist = self.max - self.min
        if i == 0:
            i = dist // 2

        if self.is_enum():
            if not any(self.values):
                return str(self.min)
            return list(self.values.keys())[i % len(self.values)]

        if self.is_fixed_size():
            i = (i % dist) + self.min
            if self.is_alias():
                return f'{self.storage}({i})'
            return str(i)

        assert self.storage.startswith('[]'), self.name
        if self.storage == '[]byte':
            b_val = ",".join(f'0x{(j % dist + self.min):0x}'
                             for j in range(min(i, 4, self.max)))
        elif self.storage == '[]uint16':
            b_val = ",".join(f'0x{(j % dist + self.min):0x}'
                             for j in range(min(i, 4, self.max)))
            b_val += b_val
        else:
            raise Error('unknown storage')
        return f'{self.storage}{{{b_val}}}'


@dataclass
class FieldSpec:
    """A field spec defines a field in a parameter or message."""
    name: str  # the name to use in structs
    type: DataType  # backing type; for arrays, the type of a single element
    padding: bool = False
    is_array: bool = False
    length: int = 0  # 0 if has a length prefix, -1 if all remaining bytes, otherwise an array length

    description: Optional[str] = None  # for doc comments

    min: Optional[int] = None  # currently unused; eventually might be used for validation
    max: Optional[int] = None  # currently unused

    partial: Optional[bool] = False  # if the next field shares its bits with this one
    bit: Optional[int] = 0  # starting bit position, where 0=MSB ("leftmost")

    # test values - set to sensible values after parsing YAML if not defined explicitly
    test_length: Optional[int] = 0  # num items in generated arrays (if not fixed-size)

    @classmethod
    def from_yaml(cls, y: YML, types: Dict[str, DataType]) -> 'FieldSpec':
        if 'type' not in y:
            raise MissingPropError(['type'], y)

        y['name'] = y.get('name', y['type'])

        typ = y['type']
        if typ.startswith('[]'):
            y['is_array'] = True
            typ = typ[2:]

        try:
            y['type'] = types[typ]
        except KeyError:
            raise DefinitionError(y, f'unknown type {typ} in field spec')

        fs = cls(**y)

        if fs.partial and fs.type.bits == 8:
            raise DefinitionError(y, f'fields with partial widths must have fewer than 8 final bits')

        if fs.is_array and 'test_length' not in y:
            if fs.length > 0:
                fs.test_length = fs.length
            elif fs.max and fs.max > 0:
                fs.test_length = fs.max
            else:
                fs.test_length = 20 if fs.type.size == 1 else 4

        return fs

    @property
    def min_size(self) -> int:
        """Returns the minimum number of bytes this field needs in an LLRP message."""
        if self.type.name in ('string', 'bitArray'):
            return 2
        if self.is_array:
            if self.length == 0:
                return 2
            if self.length == -1:
                return 0
            return self.length * self.type.size
        return self.type.size

    def is_fixed_size(self) -> bool:
        """Returns True if this field has a fixed size."""
        return ((not self.is_array and self.type.is_fixed_size()) or
                (self.is_array and self.length > 0))

    def var_name(self, struct_name: str = '') -> str:
        """Returns the variable name used to reference this field."""
        if self.padding:
            return '_'
        if struct_name.startswith('*'):
            return struct_name
        prefix = struct_name + '.' if struct_name else ''
        return f'{prefix}{self.name}'

    def struct_field(self) -> str:
        """Returns the field type as it should appear in a struct definition."""
        if self.type.name == 'bitArray':
            return f'{self.name}NumBits uint16\n{self.name} []byte'
        if self.type.name == 'string':
            return f'{self.name} string'
        return f'{self.name} {"[]" if self.is_array else ""}{self.type.name}'

    def test_field(self) -> str:
        """Returns a string for a new instance of this type, for use in tests."""
        if self.is_array:
            items = ",".join(self.type.test_instance(i) for i in range(self.test_length))
            return f'[]{self.type.name}{{{items}}}'
        if self.max is not None and self.min is not None:
            dist = self.max - self.min
            return f'{self.type.test_instance(max(self.min, dist // 2))}'
        if self.max is not None:
            return f'{self.type.test_instance(self.max)}'
        if self.min is not None:
            return f'{self.type.test_instance(self.min)}'
        return f'{self.type.test_instance(self.test_length)}'

    def value(self, begin: Union[int, str] = 0, end: Union[int, str] = '') -> str:
        """Return an expression to extract data[begin:end] and cast to this type."""
        if begin:
            from_exp = f'data[{begin}:{end}]'
        elif end:
            from_exp = f'data[:{end}]'
        else:
            from_exp = 'data'

        if self.type.size == 1:
            from_exp = f'data[{begin}]'
        elif self.type.size == 2:
            from_exp = f'binary.BigEndian.Uint16({from_exp})'
        elif self.type.size == 4:
            from_exp = f'binary.BigEndian.Uint32({from_exp})'
        elif self.type.size == 8:
            from_exp = f'binary.BigEndian.Uint64({from_exp})'

        bit_size = 8 * (self.type.size - 1) + self.type.bits
        last_bit = self.bit + bit_size
        downshift = self.type.size * 8 - last_bit
        if bit_size == 1 and self.type == 'bool':
            from_exp += f'& 0x{1 << ((self.type.size * 8) - self.bit - 1):02x}'
        else:
            if downshift != 0:
                from_exp += f'>>{downshift}'
            if self.bit != 0:
                from_exp += f'&{1 << bit_size - 1}'

        # determine if we need to cast
        if (self.type.name.startswith('uint') or
                (self.type.is_alias() and self.type.storage.startswith('uint'))):
            return from_exp
        if self.type.name == 'bitArray':
            return from_exp
        if self.type.name == 'bool':
            return f'{from_exp} != 0'
        return f'{self.type.name}({from_exp})'

    @contextmanager
    def write_unmarshal_arr(self, w, var: str, pos: int) -> str:
        """Writes unmarshaling code for an array type.
        yields an expression to reslice data past what it read."""
        data = f'data[{pos}:]' if pos else 'data'

        # for fixed size arrays
        if self.length > 0:
            if self.type.name == 'string':
                w.write(f'{var} = string(data[{pos if pos else ""}:{pos + self.length}])')
                yield f'{pos + self.length}'
                return

            w.write(f'{var} = make([]{self.type.name}, {self.length})')
            if self.type.can_copy():
                w.write(f'copy({var}, {data})')
            else:
                with w.block(f'for i := 0; i < {self.length * self.type.size}; i++'):
                    w.write(f'{var}[i] = {self.value("i")}')
            yield f'{pos + self.length * self.type.size}'
            return

        n_elems = f'binary.BigEndian.Uint16({data})'
        data = f'data[{pos + 2}:]'
        pos += 2

        # bit arrays require special length handling
        if self.type.name == 'bitArray':
            w.write(f'{var}NumBits = {n_elems}')

            w.comment('Go right shift on signed ints is arithmetic, not logical')
            n_bytes = f'1+((int({var}NumBits)-1)>>3)'
            with w.condition(f'nBytes := {n_bytes}; nBytes > len({data})'):
                w.reterr(f'{self.name} (bit array) declares it has %d bits (%d bytes), '
                         f'but only %d bytes are available',
                         [f'{var}NumBits', 'nBytes', f'len({data})'])
                w.ifelse(f'nBytes != 0')
                w.write(f'{var} = make([]byte, nBytes)')
                w.write(f'copy({var}, {data})')
                yield f'nBytes+{pos}'
            return

        # strings implicitly copy bytes during type conversion from []byte
        if self.type.name == 'string':
            with w.condition(f'strLen := int({n_elems}); strLen > len({data})'):
                w.reterr(f'{self.name} (string) declares it has %d bytes, '
                         f'but only %d bytes are available', ['strLen', f'len({data})'])
                w.ifelse('strLen != 0')
                w.write(f'{var} = string(data[{pos}:strLen+{pos}])')
                yield f'strLen+{pos}'
            return

        # all other array types
        typ = '[]' + self.type.name  # used in error strings
        sz = self.type.size
        mul = f'*{sz}' if sz != 1 else ''

        cond = f'arrLen := int({n_elems}); arrLen > len({data})'
        if sz > 1:
            w.comment('cast the len check to int64 to prevent overflow issues')
            cond = f'arrLen := int({n_elems}); int64(arrLen){mul} > int64(len({data}))'

        with w.condition(cond):
            w.reterr(f'{self.name} ({typ}) declares it has %d{mul} bytes, '
                     f'but only %d bytes are available', ['arrLen', f'len({data})'])
            w.ifelse('arrLen != 0')

            w.write(f'{var} = make({typ}, arrLen)')
            if self.type.can_copy():
                w.write(f'copy({var}, {data})')
            elif self.type.size == 1:
                with w.block(f'for i := 0; i < arrLen; i++'):
                    w.write(f'{var}[i] = {self.value(f"i+{pos}")}')
            else:
                with w.block(f'for i, pos := 0, {pos}; i < arrLen; i, pos = i+1, pos+{sz}'):
                    w.write(f'{var}[i] = {self.value("pos")}')

            yield f'arrLen{mul}+{pos}'

    def write_unmarshal(self, w, reslice: bool, prefix: str = 'pt', pos: int = 0):
        """Outputs Go code to unmarshal this fields.
        If is_last is False and this is not a fixed size type,
        it'll writes an expression to reslice the data.
        """
        if self.padding:
            if reslice:
                w.write(f'data = data[{pos + self.min_size}:]')
            return

        var = self.var_name(prefix)
        if self.is_fixed_size() and not self.is_array:
            w.write(f'{var} = {self.value(pos)}')
            if reslice:
                w.write(f'data = data[{pos + self.min_size}:]')
            return

        if self.length >= 0:
            with self.write_unmarshal_arr(w, var, pos) as pos_exp:
                if reslice:
                    w.write(f'data = data[{pos_exp}:]')
                    w.ifelse()
                    w.write(f'data = data[{pos + 2}:]')
            return

        # consume the rest of the data
        data, d_len = 'data', 'len(data)'
        if pos > 0:
            data = f'data[{pos}:]'
            d_len = f'len(data)-{pos}'

        with w.condition(f'{d_len} == 0'):
            w.write('return nil')
        w.write(f'{var} = make([]byte, {d_len})')
        w.write(f'copy({var}, {data})\n')
        if reslice:
            w.write(f'data = data[{pos + self.min_size}:]')

    def value_bytes(self, name: str) -> str:
        if self.padding:
            return ','.join(['0x00'] * self.type.size)

        if self.is_array:
            if self.length == 0:
                # TODO: handle overflow checking
                return f'byte(len({name})>>8),byte(len({name}))'
            elif self.length > 0:
                return ','.join([f'{name}[{i}]' for i in range(self.length)])
            else:
                return ''

        if self.is_fixed_size():
            if self.type.name in ('byte', 'uint8'):
                if name[0] == '*':
                    return f'byte({name})'
                return name
            if self.type.name == 'bool':
                if name[0] == '*':
                    return f'b2b(bool({name}))'
                return f'b2b({name})'
            if self.type.size == 1:
                return f'byte({name})'
            return ' ' + ','.join([
                f'byte({name}{f">>{8 * (i - 1)}" if i - 1 > 0 else ""})'
                for i in range(self.type.size, 0, -1)])

        return ''


@dataclass
class ParamSpec:
    """ParamSpec specifies the parameters permitted in a certain context,
    and under what conditions."""
    name: str  # name used for a struct; may be omitted from yaml to use the parameter name
    param_name: str  # name of an LLRP parameter type; called 'type' in the YAML
    optional: bool = False  # parameter is "0-1" or "0-n"
    repeatable: bool = False  # parameter is "0-n" or "1-n"
    air_protocol: Optional[str] = None  # if this parameter is tied to a specific air protocol...
    p_def: 'Container' = None  # parameter definition; set after parameters are read
    group: Optional[str] = None  # groups are mutually exclusive
    version: Optional[int] = 1

    description: Optional[str] = None

    def is_fixed_size(self) -> bool:
        """Returns true if this parameter is always a fixed size."""
        return self.p_def.fixed_size and self.exactly_one

    @classmethod
    def from_yaml(cls, y: YML) -> 'ParamSpec':
        if 'type' not in y:
            raise MissingPropError(['type'], y)
        if 'name' not in y:
            y['name'] = y['type']
            if y.get('repeatable') and not y['name'] == 'Custom':
                n = y['name']
                if n.endswith('Data') or n[-1] == 's':
                    pass
                elif n.endswith('y'):
                    y['name'] = n[:-1] + 'ies'
                else:
                    y['name'] += 's'
        y['param_name'] = y['type']
        del y['type']
        try:
            return ParamSpec(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    def struct_field(self) -> str:
        if self.repeatable:
            return f'{self.name} []{self.p_def.type_name}'
        return ''

    def tlv_len_check(self, w: GoWriter):
        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
        with w.condition('int(subLen) > len(data)'):
            w.reterr(f'Param{self.param_name} '
                     'says it has %d bytes, but only %d bytes remain',
                     ['subLen', 'len(data)'])

    @property
    def exactly_one(self) -> bool:
        return not (self.optional or self.repeatable or self.group)


@dataclass
class Container:
    """Container mostly represents Parameters, but Messages are nearly identical,
    so it serves double duty."""
    name: str
    type_id: int  # message or parameter number
    short: str = 'p'  # string to use as method receivers

    fields: List[FieldSpec] = dataclasses.field(default_factory=list)
    parameters: List[ParamSpec] = dataclasses.field(default_factory=list)

    # derived properties
    fixed_size: bool = False  # true if no variable length fields/optional or repeatable parameters
    min_size: int = 0  # minimum required size (incl. header); absolute size if fixed_size is True
    header_size: int = 0  # only set for Parameters, not Messages
    has_required: bool = False  # true if has required parameters

    description: Optional[str] = None
    version: Optional[int] = 1

    @classmethod
    def from_yaml(cls, y, types: Dict[str, DataType]) -> 'Container':
        missing = [req for req in ('name', 'type_id') if req not in y]
        if any(missing):
            raise ValueError(f'definition missing {missing}: {y}')

        try:
            y['fields'] = [FieldSpec.from_yaml(f, types) for f in y['fields']]
        except KeyError:
            y['fields'] = []

        try:
            y['parameters'] = [ParamSpec.from_yaml(s) for s in y['parameters']]
        except KeyError:
            y['parameters'] = []

        try:
            return cls(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    @property
    def type_name(self) -> str:
        """Returns the Go type name."""
        # return GoWriter.lower_camel(self.name)
        return self.name

    @property
    def is_tlv(self) -> bool:
        return self.type_id >= 128

    @property
    def const_name(self) -> str:
        return f'Param{self.name}'

    def empty(self) -> bool:
        """Returns True if this doesn't have any fields or parameters."""
        return not (any(self.fields) or any(self.parameters))

    def can_inline(self) -> bool:
        """Returns False if this must be a struct."""
        return (len(self.fields) == 1 and len(self.parameters) == 0
                and not self.fields[0].is_array and self.fields[0].is_fixed_size())

    def write_struct(self, w):
        """Write the Parameter as a Go struct."""
        if self.can_inline():
            f = self.fields[0]
            if f.type.name == 'bitArray':
                typ = '[]byte'
            else:
                typ = f'{"[]" if f.is_array else ""}{f.type.name}'
            w.write(f'type {self.type_name} {typ}')
            return

        if self.empty():
            w.write(f'type {self.type_name} struct {{}}\n')
            return

        with w.block(f'type {self.type_name} struct'):
            for f in self.fields:
                if f.padding:
                    continue
                if f.description:
                    w.comment(f.description)
                w.write(f.struct_field())

            for p in self.parameters:
                if p.repeatable:
                    w.write(f'{p.name} []{p.p_def.type_name}')
                elif p.optional:
                    w.write(f'{p.name} *{p.p_def.type_name}')
                else:
                    w.write(f'{p.name} {p.p_def.type_name}')

    def should_check_leftover(self) -> bool:
        """Returns True if after unmarshaling, we should check for remaining data."""
        if self.fixed_size:
            return False
        if not any(self.parameters):
            if not any(self.fields):
                return False
            return self.fields[-1].length != -1
        return True

    def write_unmarshal(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) UnmarshalBinary(data []byte) error'):
            self.write_unmarshal_body(w)

            if self.should_check_leftover():
                with w.condition('len(data) > 0'):
                    w.reterr(f'finished reading {self.name}, '
                             'but an unexpected %d bytes remain', ['len(data)'])
            w.write('return nil')

    def write_marshal(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) MarshalBinary() ([]byte, error)'):
            self.write_marshal_body(w)

    def write_new_test_instance(self, w: GoWriter, assignment: str, instances: int = 1, end: str = ''):
        """Write a 'name := value' expression that assigns a new test instance of this type."""
        if self.can_inline():
            w.write(f'{assignment} {self.type_name}({self.fields[0].type.test_instance(0)}){end}', auto_break=True)
            return

        if instances == 1:
            with w.block(f'{assignment} {self.type_name}', after_end=end + '\n'):
                self.write_literal(w)
            return

        with w.block(f'{assignment} []{self.type_name}', after_end=end + '\n'):
            with w.block(after_end=end + '\n'):
                self.write_literal(w)

    def write_literal(self, w: GoWriter):
        # write a struct literal
        for f in self.fields:
            if f.padding:
                continue
            if f.type.name == 'bitArray':
                w.write(f'{f.name}NumBits: 32,')
            w.write(f'{f.name}: {f.test_field()},')

        for (optional, g_name), p_group in groupby(self.parameters, lambda p: (p.optional, p.group)):
            if optional:
                continue
            if g_name:
                p = list(p_group)[0]
                p.p_def.write_new_test_instance(
                    w, f'{p.name}:', instances=2 if p.repeatable else 1, end=',')
            else:
                for p in p_group:
                    p.p_def.write_new_test_instance(
                        w, f'{p.name}:', instances=2 if p.repeatable else 1, end=',')

    def write_round_trip_test(self, w: GoWriter):
        self.write_new_test_instance(w, f'{self.short} :=')

        w.write(f'b, err := {self.short}.MarshalBinary()')
        with w.condition('err != nil'):
            w.write('t.Fatalf("%+v", err)')

        w.write(f'var {self.short}2 {self.type_name}')
        with w.condition(f'err := {self.short}2.UnmarshalBinary(b); err != nil'):
            w.write(f't.Errorf("%+v\\n%# 02x\\n%+v\\n%+v", err, b, {self.short}, {self.short}2)')

        with w.condition(f'!reflect.DeepEqual({self.short}, {self.short}2)'):
            w.write(f't.Errorf("mismatch:\\n%# 02x\\n%+v\\n%+v", b, {self.short}, {self.short}2)')

    def write_get_header(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) getHeader() paramHeader'):
            self.write_get_header_body(w)

    def write_get_header_body(self, w: GoWriter):
        f_sizes = []
        min_f_size = 0
        for f in self.fields:
            if f.is_fixed_size():
                if f.partial:
                    continue
                else:
                    min_f_size += f.min_size
            elif f.type.name == 'bitArray':
                min_f_size += 2
                f_sizes += [f'uint16(((int({f.var_name(self.short)}NumBits)-1)>>3)+1)']
            elif f.length == -1: # all remaining data, so no length header
                f_sizes += [f'uint16('
                            f'len({f.var_name(self.short)})'
                            f'{f"*{f.type.size}" if f.type.size > 1 else ""})']
            else:
                min_f_size += 2
                f_sizes += [f'uint16('
                            f'len({f.var_name(self.short)})'
                            f'{f"*{f.type.size}" if f.type.size > 1 else ""})']

        single_inst = [p for p in self.parameters if p.exactly_one]
        zero_or_one = [p for p in self.parameters if p.optional and not p.repeatable]
        repeatable = [p for p in self.parameters if p.repeatable]

        n_params = [str(len(single_inst))] if any(single_inst) else []
        if any(not p.exactly_one for p in self.parameters):
            n_params += [f'len({self.short}.{p.name})' for p in repeatable]
            if any(n_params):
                w.write(f'nParams := {"+ ".join(n_params)}', auto_break=True)
            elif any(zero_or_one):
                w.write('nParams := 0')

            for p in zero_or_one:
                with w.condition(f'{self.short}.{p.name} != nil'):
                    w.write(f'nParams++')
            blk = 'ph := paramHeader'
        elif any(self.parameters) and not self.fixed_size:
            blk = 'ph := paramHeader'
        else:
            blk = 'return paramHeader'

        def write_subs():
            if any(zero_or_one) or any(repeatable):
                w.write(f'subs:     make([]paramHeader, 0, nParams),')
                return

            if not (self.fixed_size and any(single_inst)):
                return

            with w.block('subs:      []paramHeader', after_end=',\n'):
                for p in single_inst:
                    w.write(f'{self.short}.{p.name}.getHeader(),')

        with w.block(blk):
            w.write(f'ParamType: {self.const_name},')
            w.write(f'data:      {self.short},')
            if any(f_sizes):
                w.write(f'sz:        {min_f_size + self.header_size} + {"+ ".join(f_sizes)},', auto_break=True)
            else:
                w.write(f'sz:        {min_f_size + self.header_size},')
            write_subs()

        if not any(self.parameters) or self.fixed_size:
            return

        if not any(zero_or_one) and not any(repeatable) and f_sizes:
            f_sizes += [f'ph.subs[{i}].sz' for i in range(len(single_inst))]
            w.write(f'ph.sz += {"+ ".join(f_sizes)}')
            w.write('return ph')
            return

        for (optional, repeatable, group_name), p_group in groupby(
                self.parameters, lambda p: (p.optional, p.repeatable, p.group)):
            p_group = list(p_group)
            if repeatable:
                for p in p_group:
                    with w.block(f'for i := range {self.short}.{p.name}'):
                        w.write(f'sh := {self.short}.{p.name}[i].getHeader()')
                        w.write(f'ph.sz += sh.sz')
                        w.write(f'ph.subs = append(ph.subs, sh)')
            elif optional:
                for p in p_group:
                    with w.condition(f'{self.short}.{p.name} != nil'):
                        w.write(f'sh := {self.short}.{p.name}.getHeader()')
                        w.write(f'ph.sz += sh.sz')
                        w.write(f'ph.subs = append(ph.subs, sh)')
            elif group_name is not None:
                with w.switch():
                    for p in p_group:
                        if p.p_def.can_inline():
                            cond = f'{self.short}.{p.name} != 0'
                        else:
                            cond = ' && '.join([
                                f'{f.var_name(f"{self.short}.{p.name}")} ' +
                                ("!= 0" if not f.is_fixed_size else "!= nil")
                                for f in p.p_def.fields])
                        with w.case(cond):
                            w.write(f'ph.subs = append(ph.subs, {self.short}.{p.name}.getHeader())')
                            w.write('ph.sz += ph.subs[len(ph.subs)-1].sz')
            else:
                for p in p_group:
                    w.write(f'ph.subs = append(ph.subs, {self.short}.{p.name}.getHeader())')
                    w.write('ph.sz += ph.subs[len(ph.subs)-1].sz')

        w.write(f'return ph')

    def write_encode(self, w: GoWriter):
        self.write_get_header(w)

        with w.block(f'func ({self.short} *{self.type_name}) EncodeFields(w io.Writer) error'):
            self.write_encode_body(w)

    def value_bytes(self) -> str:
        s = self.short if not self.can_inline() else '*' + self.short
        vb = [f'{f.value_bytes(f.var_name(s))},' for f in self.fields]
        return ''.join(vb)

    def write_encode_body(self, w: GoWriter):
        s = self.short if not self.can_inline() else '*' + self.short
        for fxd, f_group in groupby(self.fields, lambda f: f.is_fixed_size()):
            if fxd:
                f_group = list(f_group)
                vb = []
                for shift, fg in groupby(f_group, lambda f: f.type.bits != 8):
                    fg = list(fg)
                    if not shift:
                        vb += [','.join([f'{f.value_bytes(f.var_name(s))}' for f in fg])]
                        continue

                    vb += ['']
                    for f in fg:
                        put = f'{f.value_bytes(f.var_name(s))}' + \
                              f'<<{(f.type.size * 8 - f.type.bits) - f.bit}'
                        if f.partial:
                            vb[-1] += put + '|'
                        else:
                            vb[-1] += put
                            vb += ['']
                    vb = vb[:-1]

                w.write('if _,err:=w.Write([]byte{ '
                        f'{",".join(vb)}}});err!=nil{{', auto_break=True)
                w.indent()
                w.reterr(f'failed to write fields for {self.const_name}', wrap=True)
                w.dedent()
                w.write('}')
                continue

            for f in f_group:
                if f.type.name == 'bitArray':
                    with w.condition('_, err := w.Write([]byte{'
                                     f'byte({f.var_name(s)}NumBits>>8), '
                                     f'byte({f.var_name(s)}NumBits&0xFF)'
                                     '}); err != nil'):
                        w.reterr(f'failed to write length of {f.name}', wrap=True)
                    with w.condition('_, err := w.Write('
                                     f'{f.var_name(s)}, '
                                     '); err != nil'):
                        w.reterr(f'failed to write {f.name}', wrap=True)
                elif f.length != 0:
                    with w.condition(f'_, err := w.Write({f.var_name(s)}); err != nil'):
                        w.reterr(f'failed to write {f.name}', wrap=True)
                elif f.is_array or f.type.name == 'string':
                    with w.condition('_, err := w.Write([]byte{'
                                     f'byte(len({f.var_name(s)})>>8), '
                                     f'byte(len({f.var_name(s)})&0xFF)'
                                     '}); err != nil'):
                        w.reterr(f'failed to write length of {f.name}', wrap=True)
                    if f.type.name == 'string':
                        with w.condition('_, err := w.Write([]byte('
                                         f'{f.var_name(s)}'
                                         ')); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)
                    elif f.type.can_copy():
                        with w.condition('_, err := w.Write('
                                         f'{f.var_name(s)}'
                                         '); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)
                    else:
                        with w.condition('err := binary.Write(w, binary.BigEndian, '
                                         f'{f.var_name(s)}); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)

        w.write(f'return nil')

    def write_marshal_body(self, w):
        w.write('b := bytes.Buffer{}')
        w.err_check(f'encodeParams(&b, {self.short}.getHeader())', ret='nil, err')
        w.write(f'return b.Bytes()[{self.header_size}:], nil')

    def header_bytes(self) -> str:
        if self.is_tlv:
            return f'{self.type_id_bytes()},0x00,0x00'
        else:
            return f'{self.type_id_bytes()}'

    def type_id_bytes(self) -> str:
        if self.is_tlv:
            return f'0x{self.type_id >> 8:02x}, 0x{self.type_id & 0xFF:02x}'
        else:
            return f'0x{self.type_id | 0x80:02x}'

    def size_bytes(self) -> str:
        assert self.is_tlv and self.fixed_size
        return f'0x{self.min_size >> 8:02x}, 0x{self.min_size & 0xFF:02x}'

    def len_check(self, w) -> int:
        sz = self.min_size - self.header_size
        fx = str(self.fixed_size).lower()
        if sz == 0:
            if self.empty():
                w.err_check(f'hasEnoughBytes({self.const_name}, {sz}, len(data), {fx})')
            else:
                w.comment(f'{self.const_name} can be empty')
        else:
            w.err_check(f'hasEnoughBytes({self.const_name}, {sz}, len(data), {fx})')
        return sz

    def write_unmarshal_body(self, w):
        # byte length check
        known_data_len = self.len_check(w)

        # field unmarshaling
        pos = 0
        if self.can_inline():
            assert len(self.fields) == 1
            f = self.fields[0]
            w.write(f'*{self.short} = {self.type_name}({f.value()})')
        elif any(self.fields):
            for i, f in enumerate(self.fields):
                # skip reslice if this is a fixed size field,
                # or if it's the final item (i.e., last field & no params)
                reslice = ((not f.is_fixed_size()) or
                           (i == len(self.fields) - 1 and any(self.parameters)))
                f.write_unmarshal(w, reslice, self.short, pos)
                if reslice:
                    pos = 0
                elif not f.partial:
                    pos += f.min_size

                if not f.is_fixed_size():
                    known_data_len = 0
                elif not f.partial:
                    known_data_len -= f.min_size

        if not any(self.parameters):
            return

        # parameters/sub-parameters
        w.comment('sub-parameters')

        required = {p.name for p in self.parameters if not p.optional}

        groups = groupby(self.parameters, lambda x: (x.optional, x.repeatable, x.group))
        for i, ((optional, repeatable, g_name), p_group) in enumerate(groups):
            # w.comment(f'known data length: {known_data_len}')
            p_group: List[ParamSpec] = list(p_group)

            req_len = 0 if optional else min(p.p_def.min_size for p in p_group)
            if optional and not any(required):
                with w.condition(f'len(data) == 0'):
                    w.write('return nil')

            required.difference_update(p.name for p in p_group)

            # simple path: there's only a single parameter, or all in p_group are required
            if not repeatable and (len(p_group) == 1 or (g_name is None and not optional)):
                for p in p_group:
                    sub: 'Container' = p.p_def
                    if self.fixed_size:
                        assert sub.min_size <= known_data_len, f'{self.name}.{sub.name}: {sub.min_size} > {known_data_len}'

                    if not optional:
                        known_data_len = sub.header_len_check(w, known_data_len)

                    if sub.header_size == 1:
                        self.unmarshal_tv(w, p)
                    else:
                        self.unmarshal_tlv(w, p, False)

                    if not optional:
                        if sub.fixed_size:
                            known_data_len -= sub.min_size
                        else:
                            known_data_len = 0
                continue

            # otherwise, the next parameter is one of a collection of
            # repeatable, optional, intermixed, or mutually-exclusive parameters
            mut_excl = not (optional or repeatable)

            tvs = [p for p in p_group if p.p_def.header_size == 1]
            tlvs = [p for p in p_group if p.p_def.header_size == 4]

            blk = ''
            if not mut_excl:
                w.noindent(f'\nparamGroup{i}:')
                blk = f'for len(data) >= {1 if any(tvs) else 4}'

            def default_case(w: GoWriter):
                if optional or repeatable:
                    w.write(f'break paramGroup{i}')
                else:
                    w.reterr('unexpected parameter %v when unmarshaling '
                             f'Param{self.name}', ['pt'])

            has_sub_len = False
            with w.block(blk):
                if tvs and tlvs:
                    # special weirdness: the next param could be a single byte TV
                    # or it could be a TLV with a 4 byte header,
                    # so we have to check how much data is available
                    w.write('var pt ParamType')
                    with w.condition('data[0]&0x80 != 0'):
                        w.comment('TV parameter')
                        w.write('pt = ParamType(data[0]&0x7F)')
                        w.ifelse('len(data) < 4')
                        w.reterr('expecting a TLV header, but %d < 4 byte remain', ['len(data)'])
                        w.ifelse()
                        w.write('pt = ParamType(binary.BigEndian.Uint16(data))')
                elif tvs:
                    w.write('pt := ParamType(data[0]&0x7F)')
                else:
                    w.write(f'pt := ParamType(binary.BigEndian.Uint16(data))')
                    if not mut_excl:
                        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
                        has_sub_len = True
                        with w.condition('int(subLen) > len(data)'):
                            w.reterr(f'%v says it has %d bytes, but only %d bytes remain',
                                     ['pt', 'subLen', 'len(data)'])

                with w.switch('pt'):
                    self.write_cases(w, p_group, has_sub_len, default_case)

                if blk == '' or (tvs and not tlvs):
                    known_data_len -= req_len
                    continue

                if has_sub_len:
                    w.write(f'data = data[subLen:]')
                    known_data_len = 0

        return

    def unmarshal_tv(self, w: GoWriter, p: ParamSpec):
        sub = p.p_def
        if p.optional:
            blk = w.condition(f'subType := ParamType(data[0]&0x7F); '
                              f'subType == Param{sub.name}')
        else:
            blk = w.condition(f'subType := ParamType(data[0]&0x7F); '
                              f'subType != Param{sub.name}')
        with blk:
            if not p.optional:
                w.reterr(f'expected Param{sub.name}, but found %v', ['subType'])
                w.ifelse()
            self.alloc(w, p)
            self.write_unmarshal_sub(w, p, False)

    def unmarshal_tlv(self, w, p, has_sub_len):
        sub = p.p_def
        if p.optional:
            blk = w.condition(f'subType := ParamType(binary.BigEndian.Uint16(data)); '
                              f'subType == Param{sub.name}')
        else:
            blk = w.condition(f'subType := ParamType(binary.BigEndian.Uint16(data)); '
                              f'subType != Param{sub.name}')
        with blk:
            if not p.optional:
                w.reterr(f'expected Param{sub.name}, but found %v', ['subType'])
                w.ifelse()
            has_sub_len = sub.sublen_check(w)
            self.alloc(w, p)
            self.write_unmarshal_sub(w, p, has_sub_len)
            if has_sub_len:
                w.write('data = data[subLen:]')
            else:
                w.write(f'data = data[{sub.min_size}:]')

    def write_cases(self, w: GoWriter, p_group: List[ParamSpec],
                    has_sub_len: bool = False,
                    default: Optional[Callable[[GoWriter], None]] = None):
        for p in p_group:
            sub = p.p_def
            with w.case(f'Param{sub.name}'):
                sl = has_sub_len
                if not has_sub_len:
                    sl = sub.sublen_check(w)

                self.alloc(w, p)
                self.write_unmarshal_sub(w, p, sl)
                if not has_sub_len:
                    sub.len_adv(w)

        if not default:
            return

        with w.case('default'):
            default(w)

    def write_unmarshal_sub(self, w: GoWriter, p: ParamSpec, has_sub_len: bool):
        sub = p.p_def
        sub_len = 'subLen' if has_sub_len else (sub.min_size if sub.fixed_size else '')
        if p.repeatable:
            # w.write(f'tmp := new({sub.type_name})')
            w.write(f'var tmp {sub.type_name}')
            w.err_check(f'tmp.UnmarshalBinary(data[{sub.header_size}:{sub_len}])')
            w.write(f'{self.short}.{p.name} = append({self.short}.{p.name}, tmp)')
            return

        if sub.can_inline():
            exp = sub.fields[0].value(sub.header_size)
            w.write(f'{"*" if p.optional else ""}{self.short}.{p.name} = {sub.type_name}({exp})')
        else:
            w.err_check(f'{self.short}.{p.name}.UnmarshalBinary(data[{sub.header_size}:{sub_len}])')

    def alloc(self, w: GoWriter, p: ParamSpec):
        if p.optional and not p.repeatable:
            w.write(f'{self.short}.{p.name} = new({p.p_def.type_name})')

    def len_adv(self, w: GoWriter):
        """Write the code to reslice the data array past this parameter.
        If it's a TLV, subLen must be in scope and equal to it's header's claimed length.
        If it's a TV, it must be fixed size."""
        if self.header_size == 1:
            assert self.fixed_size
            w.write(f'data = data[{self.min_size}:]')
        else:
            w.write(f'data = data[subLen:]')

    def header_len_check(self, w: GoWriter, known_len: int = 0) -> int:
        if self.fixed_size and self.min_size > known_len:
            w.err_check(f'hasEnoughBytes({self.const_name}, {self.min_size}, len(data), false)')
            return self.min_size
        elif self.header_size > known_len:
            w.err_check(f'hasEnoughBytes({self.const_name}, {self.header_size}, len(data), false)')
            return self.header_size
        return known_len

    def sublen_check(self, w: GoWriter) -> bool:
        """Write a check that ensures there's at least as many bytes as the header claims.
        Only applies to TLVs; TVs simply return without writing anything."""

        '''
        if self.fixed_size:
            with w.condition(f'len(data) < {self.min_size}'):
                w.reterr(f'{self.const_name} '
                         f'needs {self.min_size} bytes, but only %d bytes remain',
                         ['len(data)'])
            return False
        '''
        if self.header_size == 1:
            return False
        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
        with w.condition('int(subLen) > len(data)'):
            w.reterr(f'{self.const_name} '
                     'says it has %d bytes, but only %d bytes remain',
                     ['subLen', 'len(data)'])
        return True

    def write_tests(self, w: GoWriter):
        with w.block(f'func Test{self.name}_roundTrip(t *testing.T)'):
            self.write_round_trip_test(w)


@dataclass
class Message(Container):
    """Overrides Container with things that Messages actually do differently than Parameters."""
    response_to: Optional[int] = None  # id of the message this is a response to

    @classmethod
    def from_yaml(cls, y, types: Dict[str, DataType]) -> 'Message':
        y['short'] = y.get('short', 'm')  # change the name of the method receiver
        # noinspection PyTypeChecker
        m: Message = super(Message, cls).from_yaml(y, types)
        return m

    def can_inline(self) -> bool:
        return False  # never inline messages

    @property
    def const_name(self) -> str:
        return self.name

    def len_check(self, w) -> int:
        if self.empty():
            w.comment(f'{self.const_name} is a header-only message')
            with w.condition('len(data) > 0'):
                w.reterr(f'{self.const_name} should be empty, but has %d bytes', ['len(data)'])
            w.write('return nil')
            return 0

        if self.fixed_size:
            with w.condition(f'len(data) != {self.min_size}'):
                w.reterr(f'{self.const_name} should length should be exactly {self.min_size}, '
                         f'but is %d', ['len(data)'])
        else:
            with w.condition(f'len(data) < {self.min_size}'):
                w.reterr(f'{self.const_name} length should be at least {self.min_size}, '
                         f'but is %d', ['len(data)'])
        return self.min_size if self.min_size is not None else 0

    def write_unmarshal(self, w):
        if any(self.fields) or any(self.parameters):
            super().write_unmarshal(w)
            return

        with w.block(f'func ({self.short} *{self.type_name}) UnmarshalBinary(data []byte) error'):
            w.comment(f'{self.name} is a header-only message')
            with w.condition('len(data) > 0'):
                w.reterr(f'{self.name} should be empty, but has %d bytes', ['len(data)'])
            w.write('return nil')

    def write_get_header(self, w):
        return

    def write_marshal_body(self, w):
        w.write('b := bytes.Buffer{}')
        w.err_check(f'{self.short}.EncodeFields(&b)', ret='nil, err')
        for p in self.parameters:
            if p.repeatable:
                with w.block(f'for i := range {self.short}.{p.name}'):
                    w.err_check(f'encodeParams(&b, {self.short}.{p.name}[i].getHeader())', ret='nil, err')
            elif p.optional:
                with w.condition(f'{self.short}.{p.name} != nil'):
                    w.err_check(f'encodeParams(&b, {self.short}.{p.name}.getHeader())', ret='nil, err')
            else:
                w.err_check(f'encodeParams(&b, {self.short}.{p.name}.getHeader())', ret='nil, err')
        w.write('return b.Bytes(), nil')

    def write_encode_body(self, w):
        if any(self.fields) or any(self.parameters):
            super().write_encode_body(w)
            return

        w.comment(f'{self.name} is a header-only message')
        w.write('return nil')

    def write_helpers(self, w: GoWriter):
        """Write the Type() and Status() interface implementations."""
        w.comment(f"Type returns this message's MessageType")
        with w.block(f'func (*{self.type_name}) Type() MessageType'):
            w.write(f'return Msg{self.name}')

        if not any(p.param_name == 'LLRPStatus' for p in self.parameters):
            return

        w.write('')
        w.comment(f"Status returns this message's LLRPStatus")
        with w.block(f'func ({self.short} *{self.type_name}) Status() LLRPStatus'):
            w.write(f'return {self.short}.LLRPStatus')


def load_data(definitions: YML) -> (Dict[str, DataType], Dict[str, Container], Dict[str, Message]):
    """Read the YAML definitions to build the object graph."""

    # predefine some types
    types: Dict[str, DataType] = {
        'bool':     DataType('bool', bits=1),
        'uint8':    DataType('uint8'),
        'byte':     DataType('byte', size=1),
        'uint16':   DataType('uint16'),
        'uint32':   DataType('uint32'),
        'uint64':   DataType('uint64'),
        'int8':     DataType('int8'),
        'int16':    DataType('int16'),
        'int32':    DataType('int32'),
        'int64':    DataType('int64'),
        'string':   DataType(
            name='string', storage='[]byte', size=1,
            description="strings in LLRP are UTF-8 values "
                        "starting with uint16 byte-length header"),
        'bitArray': DataType(
            name='bitArray', storage='[]byte', size=1,
            kind='external', min=0, max=2 ** 16 - 1,
            description="bitArrays in LLRP are a series of bits, "
                        "starting with a uint16 indicating number of bits, "
                        "padded with 0s (as LSBs) to an octet boundary.")
    }

    parameters: Dict[str, Container] = {}

    messages: Dict[str, Message] = {}

    for y in definitions['types']:
        dt = DataType.from_yaml(y, types)
        if dt.name in types:
            raise DuplicateDefError('type', dt.name, dt, types[dt.name], y)
        types[dt.name] = dt

    for y in definitions['parameters']:
        p = Container.from_yaml(y, types)
        if p.name in parameters:
            raise DuplicateDefError(f'parameter', p.name, p, parameters[p.name], y)
        parameters[p.name] = p

    for y in definitions['messages']:
        m = Message.from_yaml(y, types)
        if m.name in messages:
            raise DuplicateDefError(f'message', m.name, m, messages[m.name], y)
        messages[m.name] = m

    return types, parameters, messages


def set_param_sizes(params: Dict[str, Container]):
    """Determine the min sizes of the parameters,
    i.e., the sum of their fields' min sizes
    plus the sum of required parameters' min sizes,
    plus the parameter's header, which varies depending on its type.
    Determine if the parameter's size is in fact fixed."""
    param_sizes: Dict[str, int] = {}  # for handling parameters cycles

    def get_param_size(p: Container) -> int:
        if p.name in param_sizes:
            return param_sizes[p.name]
        param_sizes[p.name] = p.header_size

        p.min_size = (p.header_size +
                      sum(f.min_size for f in p.fields) -
                      sum(f.min_size for f in p.fields if f.partial))
        for (opt, g), p_group in groupby(p.parameters, key=lambda x: (x.optional, x.group)):
            if opt:  # skip optional groups
                for sp in p_group:
                    get_param_size(sp.p_def)
                continue
            p.min_size += min(get_param_size(sp.p_def) for sp in p_group)

        # because of the get_param_size call above, sub-parameter's fixed_size is set
        p.fixed_size = (all(f.is_fixed_size() for f in p.fields) and
                        all(sub.is_fixed_size() for sub in p.parameters))

        param_sizes[p.name] = p.min_size
        return p.min_size

    for p in params.values():
        p.header_size = 4 if p.is_tlv else 1
        for sub in p.parameters:
            try:
                sub.p_def = params[sub.param_name]
            except KeyError:
                raise MissingParamError(p, sub)

    for p in params.values():
        get_param_size(p)
        p.has_required = any(sp for sp in p.parameters if not sp.optional)


def set_msg_sizes(msgs: Dict[str, Message], params: Dict[str, Container]):
    """Set messages min sizes to the sum of their fields' min sizes
    plus the sum of required parameters' min sizes.
    Note that this doesn't include the message header."""

    for m in msgs.values():
        for p in m.parameters:
            try:
                p.p_def = params[p.param_name]
            except KeyError:
                raise MissingParamError(m, p)

        m.min_size = (sum(f.min_size for f in m.fields) -
                      sum(f.min_size for f in m.fields if f.partial) +
                      sum(p.p_def.min_size for p in m.parameters if not p.optional))
        m.fixed_size = (all(f.is_fixed_size() for f in m.fields) and
                        all(sub.is_fixed_size() for sub in m.parameters))
        m.has_required = any(sp.optional for sp in m.parameters)


def write_struct_code(w: GoWriter, types: Dict[str, DataType],
                         parameters: Dict[str, Container], messages: Dict[str, Message],
                         write_header: bool = True):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

    # write types
    for t in types.values():
        t.write_type_def(w)

    with w.paren('const'):
        for m_name, m in messages.items():
            w.write(f'Msg{m_name} = MessageType({m.type_id})')

    w.write('')
    for m in messages.values():
        w.comment(f'{m.type_name} is Message {m.type_id}, {m.name}.')
        if m.description:
            w.write('//')
            w.comment(m.description)

        m.write_struct(w)
        m.write_helpers(w)

    for p in parameters.values():
        if p.type_id == 0:
            continue

        w.comment(f'{p.type_name} is Parameter {p.type_id}, {p.name}.')
        if p.description:
            w.write('//')
            w.comment(p.description)

        p.write_struct(w)


def write_unmarshal_code(w: GoWriter, types: Dict[str, DataType],
                         parameters: Dict[str, Container], messages: Dict[str, Message],
                         write_header: bool = True):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"encoding/binary"')
            w.write('"github.com/pkg/errors"')

    w.comment("hasEnoughBytes returns an error if there aren't "
              "enough bytes to read the parameter.")
    with w.block('func hasEnoughBytes(pt ParamType, needed, got int, exact bool) error'):
        with w.condition('needed <= got'):
            w.write('return nil')
        with w.condition('exact && needed == 0'):
            w.reterr('%v must be empty, but received %d byte(s)', ['pt', 'got'])
            w.ifelse('exact')
            w.reterr('%v requires exactly %d byte(s), but received %d', ['pt', 'needed', 'got'])
        w.reterr('%v requires at least %d byte(s), but received %d', ['pt', 'needed', 'got'])

    w.write('')
    for m in messages.values():
        w.comment(f'UnmarshalBinary Message {m.type_id}, {m.name}.')
        m.write_unmarshal(w)

    for p in parameters.values():
        if p.type_id == 0:
            continue
        w.comment(f'UnmarshalBinary Parameter {p.type_id}, {p.name}.')
        p.write_unmarshal(w)


def write_marshal_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"bytes"')

        w.write('')

    w.comment("b2b converts a bool to a byte, because Go doesn't have a built-in for it.")
    with w.block('func b2b(b bool) byte'):
        with w.condition('b'):
            w.write('return 1')
        w.write('return 0')

    for m in messages.values():
        w.comment(f'MarshalBinary Message {m.type_id}, {m.name}.')
        m.write_marshal(w)

    for p in parameters.values():
        w.comment(f'MarshalBinary Parameter {p.type_id}, {p.name}.')
        p.write_marshal(w)


def write_encode_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"io"')
            w.write('"encoding/binary"')
            w.write('"github.com/pkg/errors"')

    for m in messages.values():
        w.comment(f'EncodeFields for Message {m.type_id}, {m.name}.')
        m.write_encode(w)

    for p in parameters.values():
        w.comment(f'EncodeFields for Parameter {p.type_id}, {p.name}.')
        p.write_encode(w)


def write_test_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            # w.write('"encoding/binary"')
            w.write('"testing"')
            w.write('"reflect"')
            # w.write('"github.com/pkg/errors"')

        w.write('')

    for m in messages.values():
        w.comment(f'Test Message {m.type_id}, {m.name}.')
        m.write_tests(w)

    for p in parameters.values():
        w.comment(f'Test Parameter {p.type_id}, {p.name}.')
        p.write_tests(w)


def main():
    import sys
    import argparse
    import subprocess

    parser = argparse.ArgumentParser(
        description='This is a python script to read a YAML description '
                    'of binary LLRP messages to generate Go code '
                    'that converts them to and from JSON.'
    )
    parser.add_argument('-i', '--input', help='input YAML file (default: STDIN)',
                        type=argparse.FileType('r'), default=sys.stdin)

    def add_out_file(kind: str):
        parser.add_argument(
            f'-{kind[0]}',
            f'--{kind}-file',
            default='-',
            help=f'output file for {kind} code (default: STDOUT)')

    # output files; first letter is used as argument
    add_out_file('structs')
    add_out_file('unmarshal')
    add_out_file('marshal')
    add_out_file('encode')
    add_out_file('test')

    def add_flag(name: str, help: str):
        parser.add_argument(
            f'--{name}', default=True, action='store_true',
            help=f'{help} (default true)')
        parser.add_argument(
            f'--no-{name}', dest=name, action='store_false',
            help=f"don't {help}")

    # argument names + descriptions for operations
    add_flag('structs', 'write Go structs and types')
    add_flag('unmarshal', 'write LLRP binary -> Go struct unmarshaling code')
    add_flag('marshal', 'write Go struct -> LLRP binary marshaling code')
    add_flag('encode', 'write Go -> LLRP binary fields encoder code')
    add_flag('test', 'write test code')
    add_flag('gofmt', 'pipe output through gofmt')

    parser.add_argument(
        '--parameter',
        help='output only specific parameters (repeatable)',
        action='append')

    args = parser.parse_args()

    definitions = yaml.safe_load(args.input)
    types, parameters, messages = load_data(definitions)
    set_param_sizes(parameters)
    set_msg_sizes(messages, parameters)

    @contextmanager
    def gowrite(filename: str, gofmt: bool):
        with ExitStack() as stack:
            out_file = sys.stdout
            if filename != '-':
                out_file = stack.enter_context(open(filename, 'w'))

            if gofmt:
                gofmt_proc = stack.enter_context(subprocess.Popen(
                    ['gofmt'],
                    stdin=subprocess.PIPE,
                    stdout=out_file,
                    stderr=subprocess.STDOUT,
                    encoding='utf-8'))
                out_file = gofmt_proc.stdin

            yield GoWriter(out_file)

    stdout_has_header = False
    # iterates over (argument variable, argument file, func to run if arg is true,
    # func to run if only for specific params)
    for arg, f, func, p_func in [
        (args.structs, args.structs_file, write_struct_code, Container.write_struct),
        (args.unmarshal, args.unmarshal_file, write_unmarshal_code, Container.write_unmarshal),
        (args.marshal, args.marshal_file, write_marshal_code, Container.write_marshal),
        (args.encode, args.encode_file, write_encode_code, Container.write_encode),
        (args.test, args.test_file, write_test_code, Container.write_tests),
    ]:
        if not arg:
            continue

        with gowrite(f, args.gofmt) as w:
            write_header = True
            if f == '-':
                write_header = stdout_has_header
                stdout_has_header = True

            if args.parameter:
                for p_name in args.parameter:
                    p_func(args.parameter[p_name], w)
            else:
                func(w, types, parameters, messages, write_header)
            w.write('\n')


if __name__ == '__main__':
    try:
        main()
    except DefinitionError as e:
        print(e.yml)
        raise
    except MissingParamError as e:
        print(e.parent)
        print(e.decl)
        raise

<filename>internal/llrp/generate_param_code.py
#
#  Copyright (C) 2020 Intel Corporation
#
#  SPDX-License-Identifier: Apache-2.0

# This python script reads a YAML file
# to generate Go code that unpacks binary LLRP messages
# so that they can be marshaled into JSON.
#
# Why bother doing it this way? In short, because other ways weren't good;
# it's much easier to express the LLRP message hierarchy as YAML than as Go code,
# and it's pretty easy to use that to generate parsers that are far less error prone
# than writing it by hand.
#
# At first blush, LLRP looks easy to parse,
# but when you dig in, there are a lot of rules and required context
# around how the next bytes should be interpreted.
# I tried writing it with interfaces and with reflection code,
# but it was way, way too easy to make a mistake, and I spent a ton of time
# debugging difficult to spot errors that could only be triggered at runtime.
#
# The best version came from 'hand-crafted' byte unmarshaling,
# but 1) it's still easy to mess up and 2) it would take too long to write.
#
# As I was building up a mental model around LLRP,
# I built a spreadsheet to reference the memory layouts, types, and rules.
# Looking at it, I realized it had all the need parsing info,
# so I converted to it the messages.yaml file and cleaned it up a bit,
# then wrote this code to read it in, statically check various assumptions,
# calculate some useful constraint checks from the message graph,
# and output the unmarshaling code.

try:
    import yaml
except ImportError:
    print('error: missing yaml; try "pip install pyyaml"')
    exit(1)

import dataclasses
import re
import sys
from contextlib import contextmanager, ExitStack
from dataclasses import dataclass
from itertools import groupby
from typing import Dict, List, Optional, Union, Any, Callable

YML = Dict[str, Any]  # just for type checking; otherwise does nothing


class Error(Exception):
    """Base class for errors generated by this script."""


class MissingParamError(Error):
    """Message or Parameter references an unknown Parameter."""

    def __init__(self, parent: 'Container', decl: 'ParamSpec', msg: str = None):
        if msg is None:
            msg = f'{parent.name} references unknown parameters {decl.param_name}'
        super(MissingParamError, self).__init__(msg)
        self.parent = parent
        self.decl = decl


class DefinitionError(Error):
    """Problem with the YAML definition."""

    def __init__(self, y: YML, msg=None):
        if msg is None:
            if 'name' in y:
                msg = f'failed at: {y["name"]}'
            elif 'type' in y:
                msg = f'failed at: {y["type"]}'
            else:
                msg = f'failed at: {y}'
        super(DefinitionError, self).__init__(msg)
        self.yml = y


class DuplicateDefError(DefinitionError):
    def __init__(self, kind: str, name: str, first: Any, second: Any, y: YML, msg=None):
        if msg is None:
            msg = f'duplicate {kind} definition for {name}:\n'
            f'> first instance: {first}\n'
            f'> this instance:  {second}\n'
        super(DuplicateDefError, self).__init__(y, msg)
        self.kind = kind
        self.name = name
        self.first = first
        self.second = second


class MissingPropError(DefinitionError):
    def __init__(self, missing: List[str], y: YML, msg=None):
        if msg is None:
            msg = f'missing required properties {missing}'
        super(MissingPropError, self).__init__(y, msg)


class GoWriter:
    """GoWriter outputs (mostly) formatted Go code.
    By default it writes to stdout, but you can set w to any File object
    (really, anything with a 'write' function)."""

    # map of basic types to byte lengths
    typeSizes = {
        'bool':   1,
        'int8':   1,
        'int16':  2,
        'int32':  4,
        'int64':  8,
        'uint8':  1,
        'uint16': 2,
        'uint32': 4,
        'uint64': 8,
    }

    PAREN = '()'
    BRACE = '{}'

    def __init__(self, w=None):
        import textwrap
        if w is None:
            self.w = sys.stdout
        else:
            self.w = w
        self.i = 0
        self.begin = True

        self.code = textwrap.TextWrapper(
            break_long_words=False,
            break_on_hyphens=False)

        self.buffered = None

    def noindent(self, text):
        self.w.write(text)
        self.w.write('\n')

    def buffer(self, text, pre='', auto_break=False):
        self.buffered = (text, pre, auto_break)

    def write_buffered(self):
        if self.buffered is None:
            return
        self.write(*self.buffered)

    def write(self, text, pre='', auto_break=False):
        self.buffered = None

        indent = self.i * '\t' + pre
        if not auto_break:
            for line in text.splitlines():
                self.w.write(indent + line.strip() + '\n')
            return

        requote = pre != '// '

        def write_paragraph(p):
            self.code.width = 70 - len(indent) if requote else 90 - len(indent)
            open_quote = False
            long_return = False
            for line in self.code.wrap(p):
                if long_return:
                    long_return = False
                    self.w.write(' ')
                else:
                    self.w.write(indent)

                if open_quote and requote:
                    self.w.write('"')

                self.w.write(line)

                if requote and line.count('"') & 1 == 1:
                    open_quote = not open_quote

                if open_quote:
                    self.w.write(' " +')

                if requote and line.strip() == 'return':
                    long_return = True
                else:
                    self.w.write('\n')

        paragraphs = text.split('\n')
        for p in paragraphs[:-1]:
            write_paragraph(p)
            self.w.write(pre + '\n')
        if paragraphs[-1].strip() != '':
            write_paragraph(paragraphs[-1])

    def comment(self, text, auto_break=True):
        self.write(text, pre='// ', auto_break=auto_break)

    @staticmethod
    def lower_camel(s: str) -> str:
        """Convert ABCCamelCaseString to abcCamelCaseString.
        Useful for creating unexported type names."""
        if len(s) <= 1:
            return s.lower()
        m = re.match(r'[A-Z]+', s)
        if not m or m.end() == len(s):
            return s.lower()

        if m.end() == 1:
            return s[0].lower() + s[1:]
        if s[m.end()].isalpha():
            return s[:m.end() - 1].lower() + s[m.end() - 1:]
        else:
            return s[:m.end()].lower() + s[m.end():]

    def indent(self):
        """Increase the indentation level for all subsequent output."""
        self.i += 1
        self.begin = True

    def dedent(self):
        """Decrease the indentation level fro all subsequent output."""
        self.i = max(self.i - 1, 0)
        self.begin = True

    @contextmanager
    def block(self, before_start: str = '', style=BRACE, after_end: str = '\n'):
        """Returns a context within which writes are indented inside braces,
        defined by the style. By default, this looks like:

            {
                <output written within context>
            }
            <next write goes here>

        Use like any other python context. E.g., this code:
            with writer.block('const', style='()'):
                for i in range(3):
                    w.write(f'myNum{i} = 1<<{i}')
        has this output:
            const (
                myNum0 = 1<<0
                myNum1 = 1<<1
                myNum2 = 1<<2
            )
        """
        if len(style) != 2:
            raise Error('style must have length 2')
        self.write(f'{before_start.strip()} {style[0]}'.strip())
        self.indent()
        yield
        self.dedent()
        self.write(f'{style[1]}{after_end}')

    @contextmanager
    def paren(self, before_start: str = ''):
        """Returns a context within which writes are indented inside parentheses."""
        with self.block(before_start, style=GoWriter.PAREN):
            yield

    @contextmanager
    def condition(self, cond: str):
        """Returns a context within which writes are indented inside an if (cond) block."""
        with self.block(before_start=f'if {cond}'):
            yield

    def ifelse(self, cond: Optional[str] = None):
        """Write the '} else {' or '} else if (condition) {' in an if...else block."""
        self.dedent()
        if cond:
            self.write(f'}} else if {cond} {{')
        else:
            self.write('} else {')
        self.indent()

    @contextmanager
    def switch(self, before_start: str = ''):
        """Returns a context for writing switch cases inside curly braces.
        See paren for more information."""
        with self.block(f'switch {before_start}'):
            yield

    @contextmanager
    def case(self, case: str):
        if case == 'default':
            self.write(f'default:')
        else:
            self.write(f'case {case}:')
        self.indent()
        yield
        self.dedent()

    def err_check(self, cond, params: Optional[List[str]] = None, ret: Optional[str] = 'err'):
        """Write 'if err := (cond); err != nil {\n return err\n }.
        Optionally, set other parameters and/or change the return value:
            if params[0], params[1], err := cond; err != nil {\n return ret\n}
        """
        p_list = ['err'] if not params else params + ['err']
        with self.block(f'if {", ".join(p_list)} := {cond}; err != nil'):
            if ret is not None:
                self.write('return ' + ret)

    def reterr(self, msg: Optional[str], params: Optional[List[str]] = None, wrap: bool = False):
        """Writes code to return an error, using 'return err' or one of
        New, Errorf, Wrap, or Wrapf if there are params and/or wrap is True."""
        if msg and (msg.count('%') != (len(params) if params else 0)):
            raise Error(f'got "{params}", but need {msg.count("%")} params for this error: "{msg}"')

        if not msg:
            assert params is None and wrap is False
            self.write('return err')
            return

        msg = msg.replace('\n', '" +\n"')
        if wrap and params:
            self.write(f'return errors.Wrapf(err, "{msg}", {", ".join(params)})', auto_break=True)
        elif wrap and not params:
            self.write(f'return errors.Wrap(err, "{msg}")')
        elif params:
            self.write(f'return errors.Errorf("{msg}", {", ".join(params)})', auto_break=True)
        else:
            self.write(f'return errors.New("{msg}")')


@dataclass
class DataType:
    """DataType backs field types, such as built-ins, flags, enumerations, and aliases."""
    storage: str  # underlying basic type (e.g., uint8)
    name: str = ''  # type name must be unique to the Go package
    prefix: str = ''  # when generating code, prefix is attached to enum/flag names
    kind: str = 'external'  # flag, enumeration, or alias, or external
    values: Optional[Dict[str, int]] = None  # maps name -> value, or implicitly name -> iota
    size: int = None  # storage byte size; for arrays, the byte size of a single element

    bits: int = 8  # number of bits in the final byte; usually should be 8, but occasionally not

    description: Optional[str] = None  # for outputting doc comments

    # used for testing; set to reasonable values if not specified in YAML
    min: Optional[int] = None
    max: Optional[int] = None

    @classmethod
    def from_yaml(cls, y: YML, types: Dict[str, 'DataType']) -> 'DataType':
        missing = [req for req in ('name', 'storage') if req not in y]
        if any(missing):
            raise MissingPropError(missing, y)

        if 'kind' not in y:
            y['kind'] = 'alias'

        try:
            return cls(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    def __post_init__(self):
        if self.name == '':
            self.name = self.storage

        if self.size is None:
            if self.storage.startswith('uint'):
                self.size = int(self.storage[4:]) // 8
            elif self.storage.startswith('int'):
                self.size = int(self.storage[3:]) // 8
            elif self.storage in ('bool', 'byte'):
                self.size = 1
            else:
                raise Error(f"the size of {self.name} ({self.storage}) isn't known")

        assert self.size > 0, self.name
        bts = 8 * self.size - 8 + self.bits
        if self.storage.startswith('int'):
            self.min = self.min if self.min is not None else -(1 << (bts - 1))
            self.max = self.max if self.max is not None else (1 << (bts - 1)) - 1
        else:
            self.min = self.min or 0
            self.max = self.max if self.max is not None else 1 << bts
        assert self.min is not None and self.max is not None, self.name

        if self.kind in ('alias', 'external'):
            return

        if self.kind not in ('flag', 'enum'):
            raise Error(f'unknown kind {self.kind} for type def {self.name}')

        if self.values is None:
            self.values = {}
            return self

        try:
            vals = {}
            for name, value in self.values.items():
                vals[self.prefix + name] = value
            self.values = vals
        except AttributeError:
            # allow the YAML value to use a list to map name -> position
            vals = {}
            for value, name in enumerate(self.values):
                vals[self.prefix + name] = value
            self.values = vals

        if len(self.values) > 1 << self.size * 8:
            raise Error(f'{self.name} has more items than can fit in {self.storage}')

        self.min = 0
        self.max = len(self.values)

        return self

    def write_type_def(self, w):
        if self.kind == 'external':
            return

        if self.description:
            w.comment(self.description)

        w.write(f'type {self.name}{"=" if self.is_alias() else " "}{self.storage}')
        if self.is_alias() or not any(self.values):
            return

        if len(self.values) == 1:
            name, value = list(self.values.items())[0]
            if self.kind == 'flag':
                w.write(f'const {name} = {self.name}(1 << {self.size * 8 - 1})')
            else:
                w.write(f'const {name} = {self.name}({value})')
            return

        if self.kind == 'enum':
            with w.paren('const'):
                for name, value in sorted(self.values.items(), key=lambda x: x[1]):
                    w.write(f'{name} = {self.name}({value})')
            w.write('')
            return

        assert self.kind == 'flag'
        flags = sorted(self.values.items(), key=lambda x: x[1])

        with w.paren('const'):
            w.write(f'{flags[0][0]} = {self.name}(1 << ({self.size * 8 - 1} - iota))')
            for name, _ in flags[1:]:
                w.write(f'{name}')

    def is_fixed_size(self) -> bool:
        """False for array types and True for all others."""
        return self.storage[0] != '['

    def is_alias(self) -> bool:
        return self.kind == 'alias'

    def is_enum(self) -> bool:
        return self.kind == 'enum'

    def is_external(self) -> bool:
        return self.kind == 'external'

    def can_copy(self) -> bool:
        return (self.storage in ('byte', 'uint8') and
                (self.is_alias() or self.name in ('byte', 'uint8')))

    def test_instance(self, i: int) -> str:
        """Returns valid Go code for an instance of the correct type,
        possibly using i as a seed.
        For a given positive value of i, this always returns the same value,
        but different values of i won't necessarily result in different values.
        """
        if self.name == 'string':
            return '"some arbitrary string"'
        if self.name == 'bool':
            return 'true' if i & 1 == 0 else 'false'

        assert self.max is not None and self.min is not None, self.name
        dist = self.max - self.min
        if i == 0:
            i = dist // 2

        if self.is_enum():
            if not any(self.values):
                return str(self.min)
            return list(self.values.keys())[i % len(self.values)]

        if self.is_fixed_size():
            i = (i % dist) + self.min
            if self.is_alias():
                return f'{self.storage}({i})'
            return str(i)

        assert self.storage.startswith('[]'), self.name
        if self.storage == '[]byte':
            b_val = ",".join(f'0x{(j % dist + self.min):0x}'
                             for j in range(min(i, 4, self.max)))
        elif self.storage == '[]uint16':
            b_val = ",".join(f'0x{(j % dist + self.min):0x}'
                             for j in range(min(i, 4, self.max)))
            b_val += b_val
        else:
            raise Error('unknown storage')
        return f'{self.storage}{{{b_val}}}'


@dataclass
class FieldSpec:
    """A field spec defines a field in a parameter or message."""
    name: str  # the name to use in structs
    type: DataType  # backing type; for arrays, the type of a single element
    padding: bool = False
    is_array: bool = False
    length: int = 0  # 0 if has a length prefix, -1 if all remaining bytes, otherwise an array length

    description: Optional[str] = None  # for doc comments

    min: Optional[int] = None  # currently unused; eventually might be used for validation
    max: Optional[int] = None  # currently unused

    partial: Optional[bool] = False  # if the next field shares its bits with this one
    bit: Optional[int] = 0  # starting bit position, where 0=MSB ("leftmost")

    # test values - set to sensible values after parsing YAML if not defined explicitly
    test_length: Optional[int] = 0  # num items in generated arrays (if not fixed-size)

    @classmethod
    def from_yaml(cls, y: YML, types: Dict[str, DataType]) -> 'FieldSpec':
        if 'type' not in y:
            raise MissingPropError(['type'], y)

        y['name'] = y.get('name', y['type'])

        typ = y['type']
        if typ.startswith('[]'):
            y['is_array'] = True
            typ = typ[2:]

        try:
            y['type'] = types[typ]
        except KeyError:
            raise DefinitionError(y, f'unknown type {typ} in field spec')

        fs = cls(**y)

        if fs.partial and fs.type.bits == 8:
            raise DefinitionError(y, f'fields with partial widths must have fewer than 8 final bits')

        if fs.is_array and 'test_length' not in y:
            if fs.length > 0:
                fs.test_length = fs.length
            elif fs.max and fs.max > 0:
                fs.test_length = fs.max
            else:
                fs.test_length = 20 if fs.type.size == 1 else 4

        return fs

    @property
    def min_size(self) -> int:
        """Returns the minimum number of bytes this field needs in an LLRP message."""
        if self.type.name in ('string', 'bitArray'):
            return 2
        if self.is_array:
            if self.length == 0:
                return 2
            if self.length == -1:
                return 0
            return self.length * self.type.size
        return self.type.size

    def is_fixed_size(self) -> bool:
        """Returns True if this field has a fixed size."""
        return ((not self.is_array and self.type.is_fixed_size()) or
                (self.is_array and self.length > 0))

    def var_name(self, struct_name: str = '') -> str:
        """Returns the variable name used to reference this field."""
        if self.padding:
            return '_'
        if struct_name.startswith('*'):
            return struct_name
        prefix = struct_name + '.' if struct_name else ''
        return f'{prefix}{self.name}'

    def struct_field(self) -> str:
        """Returns the field type as it should appear in a struct definition."""
        if self.type.name == 'bitArray':
            return f'{self.name}NumBits uint16\n{self.name} []byte'
        if self.type.name == 'string':
            return f'{self.name} string'
        return f'{self.name} {"[]" if self.is_array else ""}{self.type.name}'

    def test_field(self) -> str:
        """Returns a string for a new instance of this type, for use in tests."""
        if self.is_array:
            items = ",".join(self.type.test_instance(i) for i in range(self.test_length))
            return f'[]{self.type.name}{{{items}}}'
        if self.max is not None and self.min is not None:
            dist = self.max - self.min
            return f'{self.type.test_instance(max(self.min, dist // 2))}'
        if self.max is not None:
            return f'{self.type.test_instance(self.max)}'
        if self.min is not None:
            return f'{self.type.test_instance(self.min)}'
        return f'{self.type.test_instance(self.test_length)}'

    def value(self, begin: Union[int, str] = 0, end: Union[int, str] = '') -> str:
        """Return an expression to extract data[begin:end] and cast to this type."""
        if begin:
            from_exp = f'data[{begin}:{end}]'
        elif end:
            from_exp = f'data[:{end}]'
        else:
            from_exp = 'data'

        if self.type.size == 1:
            from_exp = f'data[{begin}]'
        elif self.type.size == 2:
            from_exp = f'binary.BigEndian.Uint16({from_exp})'
        elif self.type.size == 4:
            from_exp = f'binary.BigEndian.Uint32({from_exp})'
        elif self.type.size == 8:
            from_exp = f'binary.BigEndian.Uint64({from_exp})'

        bit_size = 8 * (self.type.size - 1) + self.type.bits
        last_bit = self.bit + bit_size
        downshift = self.type.size * 8 - last_bit
        if bit_size == 1 and self.type == 'bool':
            from_exp += f'& 0x{1 << ((self.type.size * 8) - self.bit - 1):02x}'
        else:
            if downshift != 0:
                from_exp += f'>>{downshift}'
            if self.bit != 0:
                from_exp += f'&{1 << bit_size - 1}'

        # determine if we need to cast
        if (self.type.name.startswith('uint') or
                (self.type.is_alias() and self.type.storage.startswith('uint'))):
            return from_exp
        if self.type.name == 'bitArray':
            return from_exp
        if self.type.name == 'bool':
            return f'{from_exp} != 0'
        return f'{self.type.name}({from_exp})'

    @contextmanager
    def write_unmarshal_arr(self, w, var: str, pos: int) -> str:
        """Writes unmarshaling code for an array type.
        yields an expression to reslice data past what it read."""
        data = f'data[{pos}:]' if pos else 'data'

        # for fixed size arrays
        if self.length > 0:
            if self.type.name == 'string':
                w.write(f'{var} = string(data[{pos if pos else ""}:{pos + self.length}])')
                yield f'{pos + self.length}'
                return

            w.write(f'{var} = make([]{self.type.name}, {self.length})')
            if self.type.can_copy():
                w.write(f'copy({var}, {data})')
            else:
                with w.block(f'for i := 0; i < {self.length * self.type.size}; i++'):
                    w.write(f'{var}[i] = {self.value("i")}')
            yield f'{pos + self.length * self.type.size}'
            return

        n_elems = f'binary.BigEndian.Uint16({data})'
        data = f'data[{pos + 2}:]'
        pos += 2

        # bit arrays require special length handling
        if self.type.name == 'bitArray':
            w.write(f'{var}NumBits = {n_elems}')

            w.comment('Go right shift on signed ints is arithmetic, not logical')
            n_bytes = f'1+((int({var}NumBits)-1)>>3)'
            with w.condition(f'nBytes := {n_bytes}; nBytes > len({data})'):
                w.reterr(f'{self.name} (bit array) declares it has %d bits (%d bytes), '
                         f'but only %d bytes are available',
                         [f'{var}NumBits', 'nBytes', f'len({data})'])
                w.ifelse(f'nBytes != 0')
                w.write(f'{var} = make([]byte, nBytes)')
                w.write(f'copy({var}, {data})')
                yield f'nBytes+{pos}'
            return

        # strings implicitly copy bytes during type conversion from []byte
        if self.type.name == 'string':
            with w.condition(f'strLen := int({n_elems}); strLen > len({data})'):
                w.reterr(f'{self.name} (string) declares it has %d bytes, '
                         f'but only %d bytes are available', ['strLen', f'len({data})'])
                w.ifelse('strLen != 0')
                w.write(f'{var} = string(data[{pos}:strLen+{pos}])')
                yield f'strLen+{pos}'
            return

        # all other array types
        typ = '[]' + self.type.name  # used in error strings
        sz = self.type.size
        mul = f'*{sz}' if sz != 1 else ''

        cond = f'arrLen := int({n_elems}); arrLen > len({data})'
        if sz > 1:
            w.comment('cast the len check to int64 to prevent overflow issues')
            cond = f'arrLen := int({n_elems}); int64(arrLen){mul} > int64(len({data}))'

        with w.condition(cond):
            w.reterr(f'{self.name} ({typ}) declares it has %d{mul} bytes, '
                     f'but only %d bytes are available', ['arrLen', f'len({data})'])
            w.ifelse('arrLen != 0')

            w.write(f'{var} = make({typ}, arrLen)')
            if self.type.can_copy():
                w.write(f'copy({var}, {data})')
            elif self.type.size == 1:
                with w.block(f'for i := 0; i < arrLen; i++'):
                    w.write(f'{var}[i] = {self.value(f"i+{pos}")}')
            else:
                with w.block(f'for i, pos := 0, {pos}; i < arrLen; i, pos = i+1, pos+{sz}'):
                    w.write(f'{var}[i] = {self.value("pos")}')

            yield f'arrLen{mul}+{pos}'

    def write_unmarshal(self, w, reslice: bool, prefix: str = 'pt', pos: int = 0):
        """Outputs Go code to unmarshal this fields.
        If is_last is False and this is not a fixed size type,
        it'll writes an expression to reslice the data.
        """
        if self.padding:
            if reslice:
                w.write(f'data = data[{pos + self.min_size}:]')
            return

        var = self.var_name(prefix)
        if self.is_fixed_size() and not self.is_array:
            w.write(f'{var} = {self.value(pos)}')
            if reslice:
                w.write(f'data = data[{pos + self.min_size}:]')
            return

        if self.length >= 0:
            with self.write_unmarshal_arr(w, var, pos) as pos_exp:
                if reslice:
                    w.write(f'data = data[{pos_exp}:]')
                    w.ifelse()
                    w.write(f'data = data[{pos + 2}:]')
            return

        # consume the rest of the data
        data, d_len = 'data', 'len(data)'
        if pos > 0:
            data = f'data[{pos}:]'
            d_len = f'len(data)-{pos}'

        with w.condition(f'{d_len} == 0'):
            w.write('return nil')
        w.write(f'{var} = make([]byte, {d_len})')
        w.write(f'copy({var}, {data})\n')
        if reslice:
            w.write(f'data = data[{pos + self.min_size}:]')

    def value_bytes(self, name: str) -> str:
        if self.padding:
            return ','.join(['0x00'] * self.type.size)

        if self.is_array:
            if self.length == 0:
                # TODO: handle overflow checking
                return f'byte(len({name})>>8),byte(len({name}))'
            elif self.length > 0:
                return ','.join([f'{name}[{i}]' for i in range(self.length)])
            else:
                return ''

        if self.is_fixed_size():
            if self.type.name in ('byte', 'uint8'):
                if name[0] == '*':
                    return f'byte({name})'
                return name
            if self.type.name == 'bool':
                if name[0] == '*':
                    return f'b2b(bool({name}))'
                return f'b2b({name})'
            if self.type.size == 1:
                return f'byte({name})'
            return ' ' + ','.join([
                f'byte({name}{f">>{8 * (i - 1)}" if i - 1 > 0 else ""})'
                for i in range(self.type.size, 0, -1)])

        return ''


@dataclass
class ParamSpec:
    """ParamSpec specifies the parameters permitted in a certain context,
    and under what conditions."""
    name: str  # name used for a struct; may be omitted from yaml to use the parameter name
    param_name: str  # name of an LLRP parameter type; called 'type' in the YAML
    optional: bool = False  # parameter is "0-1" or "0-n"
    repeatable: bool = False  # parameter is "0-n" or "1-n"
    air_protocol: Optional[str] = None  # if this parameter is tied to a specific air protocol...
    p_def: 'Container' = None  # parameter definition; set after parameters are read
    group: Optional[str] = None  # groups are mutually exclusive
    version: Optional[int] = 1

    description: Optional[str] = None

    def is_fixed_size(self) -> bool:
        """Returns true if this parameter is always a fixed size."""
        return self.p_def.fixed_size and self.exactly_one

    @classmethod
    def from_yaml(cls, y: YML) -> 'ParamSpec':
        if 'type' not in y:
            raise MissingPropError(['type'], y)
        if 'name' not in y:
            y['name'] = y['type']
            if y.get('repeatable') and not y['name'] == 'Custom':
                n = y['name']
                if n.endswith('Data') or n[-1] == 's':
                    pass
                elif n.endswith('y'):
                    y['name'] = n[:-1] + 'ies'
                else:
                    y['name'] += 's'
        y['param_name'] = y['type']
        del y['type']
        try:
            return ParamSpec(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    def struct_field(self) -> str:
        if self.repeatable:
            return f'{self.name} []{self.p_def.type_name}'
        return ''

    def tlv_len_check(self, w: GoWriter):
        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
        with w.condition('int(subLen) > len(data)'):
            w.reterr(f'Param{self.param_name} '
                     'says it has %d bytes, but only %d bytes remain',
                     ['subLen', 'len(data)'])

    @property
    def exactly_one(self) -> bool:
        return not (self.optional or self.repeatable or self.group)


@dataclass
class Container:
    """Container mostly represents Parameters, but Messages are nearly identical,
    so it serves double duty."""
    name: str
    type_id: int  # message or parameter number
    short: str = 'p'  # string to use as method receivers

    fields: List[FieldSpec] = dataclasses.field(default_factory=list)
    parameters: List[ParamSpec] = dataclasses.field(default_factory=list)

    # derived properties
    fixed_size: bool = False  # true if no variable length fields/optional or repeatable parameters
    min_size: int = 0  # minimum required size (incl. header); absolute size if fixed_size is True
    header_size: int = 0  # only set for Parameters, not Messages
    has_required: bool = False  # true if has required parameters

    description: Optional[str] = None
    version: Optional[int] = 1

    @classmethod
    def from_yaml(cls, y, types: Dict[str, DataType]) -> 'Container':
        missing = [req for req in ('name', 'type_id') if req not in y]
        if any(missing):
            raise ValueError(f'definition missing {missing}: {y}')

        try:
            y['fields'] = [FieldSpec.from_yaml(f, types) for f in y['fields']]
        except KeyError:
            y['fields'] = []

        try:
            y['parameters'] = [ParamSpec.from_yaml(s) for s in y['parameters']]
        except KeyError:
            y['parameters'] = []

        try:
            return cls(**y)
        except TypeError as e:
            raise DefinitionError(y) from e

    @property
    def type_name(self) -> str:
        """Returns the Go type name."""
        # return GoWriter.lower_camel(self.name)
        return self.name

    @property
    def is_tlv(self) -> bool:
        return self.type_id >= 128

    @property
    def const_name(self) -> str:
        return f'Param{self.name}'

    def empty(self) -> bool:
        """Returns True if this doesn't have any fields or parameters."""
        return not (any(self.fields) or any(self.parameters))

    def can_inline(self) -> bool:
        """Returns False if this must be a struct."""
        return (len(self.fields) == 1 and len(self.parameters) == 0
                and not self.fields[0].is_array and self.fields[0].is_fixed_size())

    def write_struct(self, w):
        """Write the Parameter as a Go struct."""
        if self.can_inline():
            f = self.fields[0]
            if f.type.name == 'bitArray':
                typ = '[]byte'
            else:
                typ = f'{"[]" if f.is_array else ""}{f.type.name}'
            w.write(f'type {self.type_name} {typ}')
            return

        if self.empty():
            w.write(f'type {self.type_name} struct {{}}\n')
            return

        with w.block(f'type {self.type_name} struct'):
            for f in self.fields:
                if f.padding:
                    continue
                if f.description:
                    w.comment(f.description)
                w.write(f.struct_field())

            for p in self.parameters:
                if p.repeatable:
                    w.write(f'{p.name} []{p.p_def.type_name}')
                elif p.optional:
                    w.write(f'{p.name} *{p.p_def.type_name}')
                else:
                    w.write(f'{p.name} {p.p_def.type_name}')

    def should_check_leftover(self) -> bool:
        """Returns True if after unmarshaling, we should check for remaining data."""
        if self.fixed_size:
            return False
        if not any(self.parameters):
            if not any(self.fields):
                return False
            return self.fields[-1].length != -1
        return True

    def write_unmarshal(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) UnmarshalBinary(data []byte) error'):
            self.write_unmarshal_body(w)

            if self.should_check_leftover():
                with w.condition('len(data) > 0'):
                    w.reterr(f'finished reading {self.name}, '
                             'but an unexpected %d bytes remain', ['len(data)'])
            w.write('return nil')

    def write_marshal(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) MarshalBinary() ([]byte, error)'):
            self.write_marshal_body(w)

    def write_new_test_instance(self, w: GoWriter, assignment: str, instances: int = 1, end: str = ''):
        """Write a 'name := value' expression that assigns a new test instance of this type."""
        if self.can_inline():
            w.write(f'{assignment} {self.type_name}({self.fields[0].type.test_instance(0)}){end}', auto_break=True)
            return

        if instances == 1:
            with w.block(f'{assignment} {self.type_name}', after_end=end + '\n'):
                self.write_literal(w)
            return

        with w.block(f'{assignment} []{self.type_name}', after_end=end + '\n'):
            with w.block(after_end=end + '\n'):
                self.write_literal(w)

    def write_literal(self, w: GoWriter):
        # write a struct literal
        for f in self.fields:
            if f.padding:
                continue
            if f.type.name == 'bitArray':
                w.write(f'{f.name}NumBits: 32,')
            w.write(f'{f.name}: {f.test_field()},')

        for (optional, g_name), p_group in groupby(self.parameters, lambda p: (p.optional, p.group)):
            if optional:
                continue
            if g_name:
                p = list(p_group)[0]
                p.p_def.write_new_test_instance(
                    w, f'{p.name}:', instances=2 if p.repeatable else 1, end=',')
            else:
                for p in p_group:
                    p.p_def.write_new_test_instance(
                        w, f'{p.name}:', instances=2 if p.repeatable else 1, end=',')

    def write_round_trip_test(self, w: GoWriter):
        self.write_new_test_instance(w, f'{self.short} :=')

        w.write(f'b, err := {self.short}.MarshalBinary()')
        with w.condition('err != nil'):
            w.write('t.Fatalf("%+v", err)')

        w.write(f'var {self.short}2 {self.type_name}')
        with w.condition(f'err := {self.short}2.UnmarshalBinary(b); err != nil'):
            w.write(f't.Errorf("%+v\\n%# 02x\\n%+v\\n%+v", err, b, {self.short}, {self.short}2)')

        with w.condition(f'!reflect.DeepEqual({self.short}, {self.short}2)'):
            w.write(f't.Errorf("mismatch:\\n%# 02x\\n%+v\\n%+v", b, {self.short}, {self.short}2)')

    def write_get_header(self, w: GoWriter):
        with w.block(f'func ({self.short} *{self.type_name}) getHeader() paramHeader'):
            self.write_get_header_body(w)

    def write_get_header_body(self, w: GoWriter):
        f_sizes = []
        min_f_size = 0
        for f in self.fields:
            if f.is_fixed_size():
                if f.partial:
                    continue
                else:
                    min_f_size += f.min_size
            elif f.type.name == 'bitArray':
                min_f_size += 2
                f_sizes += [f'uint16(((int({f.var_name(self.short)}NumBits)-1)>>3)+1)']
            elif f.length == -1: # all remaining data, so no length header
                f_sizes += [f'uint16('
                            f'len({f.var_name(self.short)})'
                            f'{f"*{f.type.size}" if f.type.size > 1 else ""})']
            else:
                min_f_size += 2
                f_sizes += [f'uint16('
                            f'len({f.var_name(self.short)})'
                            f'{f"*{f.type.size}" if f.type.size > 1 else ""})']

        single_inst = [p for p in self.parameters if p.exactly_one]
        zero_or_one = [p for p in self.parameters if p.optional and not p.repeatable]
        repeatable = [p for p in self.parameters if p.repeatable]

        n_params = [str(len(single_inst))] if any(single_inst) else []
        if any(not p.exactly_one for p in self.parameters):
            n_params += [f'len({self.short}.{p.name})' for p in repeatable]
            if any(n_params):
                w.write(f'nParams := {"+ ".join(n_params)}', auto_break=True)
            elif any(zero_or_one):
                w.write('nParams := 0')

            for p in zero_or_one:
                with w.condition(f'{self.short}.{p.name} != nil'):
                    w.write(f'nParams++')
            blk = 'ph := paramHeader'
        elif any(self.parameters) and not self.fixed_size:
            blk = 'ph := paramHeader'
        else:
            blk = 'return paramHeader'

        def write_subs():
            if any(zero_or_one) or any(repeatable):
                w.write(f'subs:     make([]paramHeader, 0, nParams),')
                return

            if not (self.fixed_size and any(single_inst)):
                return

            with w.block('subs:      []paramHeader', after_end=',\n'):
                for p in single_inst:
                    w.write(f'{self.short}.{p.name}.getHeader(),')

        with w.block(blk):
            w.write(f'ParamType: {self.const_name},')
            w.write(f'data:      {self.short},')
            if any(f_sizes):
                w.write(f'sz:        {min_f_size + self.header_size} + {"+ ".join(f_sizes)},', auto_break=True)
            else:
                w.write(f'sz:        {min_f_size + self.header_size},')
            write_subs()

        if not any(self.parameters) or self.fixed_size:
            return

        if not any(zero_or_one) and not any(repeatable) and f_sizes:
            f_sizes += [f'ph.subs[{i}].sz' for i in range(len(single_inst))]
            w.write(f'ph.sz += {"+ ".join(f_sizes)}')
            w.write('return ph')
            return

        for (optional, repeatable, group_name), p_group in groupby(
                self.parameters, lambda p: (p.optional, p.repeatable, p.group)):
            p_group = list(p_group)
            if repeatable:
                for p in p_group:
                    with w.block(f'for i := range {self.short}.{p.name}'):
                        w.write(f'sh := {self.short}.{p.name}[i].getHeader()')
                        w.write(f'ph.sz += sh.sz')
                        w.write(f'ph.subs = append(ph.subs, sh)')
            elif optional:
                for p in p_group:
                    with w.condition(f'{self.short}.{p.name} != nil'):
                        w.write(f'sh := {self.short}.{p.name}.getHeader()')
                        w.write(f'ph.sz += sh.sz')
                        w.write(f'ph.subs = append(ph.subs, sh)')
            elif group_name is not None:
                with w.switch():
                    for p in p_group:
                        if p.p_def.can_inline():
                            cond = f'{self.short}.{p.name} != 0'
                        else:
                            cond = ' && '.join([
                                f'{f.var_name(f"{self.short}.{p.name}")} ' +
                                ("!= 0" if not f.is_fixed_size else "!= nil")
                                for f in p.p_def.fields])
                        with w.case(cond):
                            w.write(f'ph.subs = append(ph.subs, {self.short}.{p.name}.getHeader())')
                            w.write('ph.sz += ph.subs[len(ph.subs)-1].sz')
            else:
                for p in p_group:
                    w.write(f'ph.subs = append(ph.subs, {self.short}.{p.name}.getHeader())')
                    w.write('ph.sz += ph.subs[len(ph.subs)-1].sz')

        w.write(f'return ph')

    def write_encode(self, w: GoWriter):
        self.write_get_header(w)

        with w.block(f'func ({self.short} *{self.type_name}) EncodeFields(w io.Writer) error'):
            self.write_encode_body(w)

    def value_bytes(self) -> str:
        s = self.short if not self.can_inline() else '*' + self.short
        vb = [f'{f.value_bytes(f.var_name(s))},' for f in self.fields]
        return ''.join(vb)

    def write_encode_body(self, w: GoWriter):
        s = self.short if not self.can_inline() else '*' + self.short
        for fxd, f_group in groupby(self.fields, lambda f: f.is_fixed_size()):
            if fxd:
                f_group = list(f_group)
                vb = []
                for shift, fg in groupby(f_group, lambda f: f.type.bits != 8):
                    fg = list(fg)
                    if not shift:
                        vb += [','.join([f'{f.value_bytes(f.var_name(s))}' for f in fg])]
                        continue

                    vb += ['']
                    for f in fg:
                        put = f'{f.value_bytes(f.var_name(s))}' + \
                              f'<<{(f.type.size * 8 - f.type.bits) - f.bit}'
                        if f.partial:
                            vb[-1] += put + '|'
                        else:
                            vb[-1] += put
                            vb += ['']
                    vb = vb[:-1]

                w.write('if _,err:=w.Write([]byte{ '
                        f'{",".join(vb)}}});err!=nil{{', auto_break=True)
                w.indent()
                w.reterr(f'failed to write fields for {self.const_name}', wrap=True)
                w.dedent()
                w.write('}')
                continue

            for f in f_group:
                if f.type.name == 'bitArray':
                    with w.condition('_, err := w.Write([]byte{'
                                     f'byte({f.var_name(s)}NumBits>>8), '
                                     f'byte({f.var_name(s)}NumBits&0xFF)'
                                     '}); err != nil'):
                        w.reterr(f'failed to write length of {f.name}', wrap=True)
                    with w.condition('_, err := w.Write('
                                     f'{f.var_name(s)}, '
                                     '); err != nil'):
                        w.reterr(f'failed to write {f.name}', wrap=True)
                elif f.length != 0:
                    with w.condition(f'_, err := w.Write({f.var_name(s)}); err != nil'):
                        w.reterr(f'failed to write {f.name}', wrap=True)
                elif f.is_array or f.type.name == 'string':
                    with w.condition('_, err := w.Write([]byte{'
                                     f'byte(len({f.var_name(s)})>>8), '
                                     f'byte(len({f.var_name(s)})&0xFF)'
                                     '}); err != nil'):
                        w.reterr(f'failed to write length of {f.name}', wrap=True)
                    if f.type.name == 'string':
                        with w.condition('_, err := w.Write([]byte('
                                         f'{f.var_name(s)}'
                                         ')); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)
                    elif f.type.can_copy():
                        with w.condition('_, err := w.Write('
                                         f'{f.var_name(s)}'
                                         '); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)
                    else:
                        with w.condition('err := binary.Write(w, binary.BigEndian, '
                                         f'{f.var_name(s)}); err != nil'):
                            w.reterr(f'failed to write {f.name}', wrap=True)

        w.write(f'return nil')

    def write_marshal_body(self, w):
        w.write('b := bytes.Buffer{}')
        w.err_check(f'encodeParams(&b, {self.short}.getHeader())', ret='nil, err')
        w.write(f'return b.Bytes()[{self.header_size}:], nil')

    def header_bytes(self) -> str:
        if self.is_tlv:
            return f'{self.type_id_bytes()},0x00,0x00'
        else:
            return f'{self.type_id_bytes()}'

    def type_id_bytes(self) -> str:
        if self.is_tlv:
            return f'0x{self.type_id >> 8:02x}, 0x{self.type_id & 0xFF:02x}'
        else:
            return f'0x{self.type_id | 0x80:02x}'

    def size_bytes(self) -> str:
        assert self.is_tlv and self.fixed_size
        return f'0x{self.min_size >> 8:02x}, 0x{self.min_size & 0xFF:02x}'

    def len_check(self, w) -> int:
        sz = self.min_size - self.header_size
        fx = str(self.fixed_size).lower()
        if sz == 0:
            if self.empty():
                w.err_check(f'hasEnoughBytes({self.const_name}, {sz}, len(data), {fx})')
            else:
                w.comment(f'{self.const_name} can be empty')
        else:
            w.err_check(f'hasEnoughBytes({self.const_name}, {sz}, len(data), {fx})')
        return sz

    def write_unmarshal_body(self, w):
        # byte length check
        known_data_len = self.len_check(w)

        # field unmarshaling
        pos = 0
        if self.can_inline():
            assert len(self.fields) == 1
            f = self.fields[0]
            w.write(f'*{self.short} = {self.type_name}({f.value()})')
        elif any(self.fields):
            for i, f in enumerate(self.fields):
                # skip reslice if this is a fixed size field,
                # or if it's the final item (i.e., last field & no params)
                reslice = ((not f.is_fixed_size()) or
                           (i == len(self.fields) - 1 and any(self.parameters)))
                f.write_unmarshal(w, reslice, self.short, pos)
                if reslice:
                    pos = 0
                elif not f.partial:
                    pos += f.min_size

                if not f.is_fixed_size():
                    known_data_len = 0
                elif not f.partial:
                    known_data_len -= f.min_size

        if not any(self.parameters):
            return

        # parameters/sub-parameters
        w.comment('sub-parameters')

        required = {p.name for p in self.parameters if not p.optional}

        groups = groupby(self.parameters, lambda x: (x.optional, x.repeatable, x.group))
        for i, ((optional, repeatable, g_name), p_group) in enumerate(groups):
            # w.comment(f'known data length: {known_data_len}')
            p_group: List[ParamSpec] = list(p_group)

            req_len = 0 if optional else min(p.p_def.min_size for p in p_group)
            if optional and not any(required):
                with w.condition(f'len(data) == 0'):
                    w.write('return nil')

            required.difference_update(p.name for p in p_group)

            # simple path: there's only a single parameter, or all in p_group are required
            if not repeatable and (len(p_group) == 1 or (g_name is None and not optional)):
                for p in p_group:
                    sub: 'Container' = p.p_def
                    if self.fixed_size:
                        assert sub.min_size <= known_data_len, f'{self.name}.{sub.name}: {sub.min_size} > {known_data_len}'

                    if not optional:
                        known_data_len = sub.header_len_check(w, known_data_len)

                    if sub.header_size == 1:
                        self.unmarshal_tv(w, p)
                    else:
                        self.unmarshal_tlv(w, p, False)

                    if not optional:
                        if sub.fixed_size:
                            known_data_len -= sub.min_size
                        else:
                            known_data_len = 0
                continue

            # otherwise, the next parameter is one of a collection of
            # repeatable, optional, intermixed, or mutually-exclusive parameters
            mut_excl = not (optional or repeatable)

            tvs = [p for p in p_group if p.p_def.header_size == 1]
            tlvs = [p for p in p_group if p.p_def.header_size == 4]

            blk = ''
            if not mut_excl:
                w.noindent(f'\nparamGroup{i}:')
                blk = f'for len(data) >= {1 if any(tvs) else 4}'

            def default_case(w: GoWriter):
                if optional or repeatable:
                    w.write(f'break paramGroup{i}')
                else:
                    w.reterr('unexpected parameter %v when unmarshaling '
                             f'Param{self.name}', ['pt'])

            has_sub_len = False
            with w.block(blk):
                if tvs and tlvs:
                    # special weirdness: the next param could be a single byte TV
                    # or it could be a TLV with a 4 byte header,
                    # so we have to check how much data is available
                    w.write('var pt ParamType')
                    with w.condition('data[0]&0x80 != 0'):
                        w.comment('TV parameter')
                        w.write('pt = ParamType(data[0]&0x7F)')
                        w.ifelse('len(data) < 4')
                        w.reterr('expecting a TLV header, but %d < 4 byte remain', ['len(data)'])
                        w.ifelse()
                        w.write('pt = ParamType(binary.BigEndian.Uint16(data))')
                elif tvs:
                    w.write('pt := ParamType(data[0]&0x7F)')
                else:
                    w.write(f'pt := ParamType(binary.BigEndian.Uint16(data))')
                    if not mut_excl:
                        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
                        has_sub_len = True
                        with w.condition('int(subLen) > len(data)'):
                            w.reterr(f'%v says it has %d bytes, but only %d bytes remain',
                                     ['pt', 'subLen', 'len(data)'])

                with w.switch('pt'):
                    self.write_cases(w, p_group, has_sub_len, default_case)

                if blk == '' or (tvs and not tlvs):
                    known_data_len -= req_len
                    continue

                if has_sub_len:
                    w.write(f'data = data[subLen:]')
                    known_data_len = 0

        return

    def unmarshal_tv(self, w: GoWriter, p: ParamSpec):
        sub = p.p_def
        if p.optional:
            blk = w.condition(f'subType := ParamType(data[0]&0x7F); '
                              f'subType == Param{sub.name}')
        else:
            blk = w.condition(f'subType := ParamType(data[0]&0x7F); '
                              f'subType != Param{sub.name}')
        with blk:
            if not p.optional:
                w.reterr(f'expected Param{sub.name}, but found %v', ['subType'])
                w.ifelse()
            self.alloc(w, p)
            self.write_unmarshal_sub(w, p, False)

    def unmarshal_tlv(self, w, p, has_sub_len):
        sub = p.p_def
        if p.optional:
            blk = w.condition(f'subType := ParamType(binary.BigEndian.Uint16(data)); '
                              f'subType == Param{sub.name}')
        else:
            blk = w.condition(f'subType := ParamType(binary.BigEndian.Uint16(data)); '
                              f'subType != Param{sub.name}')
        with blk:
            if not p.optional:
                w.reterr(f'expected Param{sub.name}, but found %v', ['subType'])
                w.ifelse()
            has_sub_len = sub.sublen_check(w)
            self.alloc(w, p)
            self.write_unmarshal_sub(w, p, has_sub_len)
            if has_sub_len:
                w.write('data = data[subLen:]')
            else:
                w.write(f'data = data[{sub.min_size}:]')

    def write_cases(self, w: GoWriter, p_group: List[ParamSpec],
                    has_sub_len: bool = False,
                    default: Optional[Callable[[GoWriter], None]] = None):
        for p in p_group:
            sub = p.p_def
            with w.case(f'Param{sub.name}'):
                sl = has_sub_len
                if not has_sub_len:
                    sl = sub.sublen_check(w)

                self.alloc(w, p)
                self.write_unmarshal_sub(w, p, sl)
                if not has_sub_len:
                    sub.len_adv(w)

        if not default:
            return

        with w.case('default'):
            default(w)

    def write_unmarshal_sub(self, w: GoWriter, p: ParamSpec, has_sub_len: bool):
        sub = p.p_def
        sub_len = 'subLen' if has_sub_len else (sub.min_size if sub.fixed_size else '')
        if p.repeatable:
            # w.write(f'tmp := new({sub.type_name})')
            w.write(f'var tmp {sub.type_name}')
            w.err_check(f'tmp.UnmarshalBinary(data[{sub.header_size}:{sub_len}])')
            w.write(f'{self.short}.{p.name} = append({self.short}.{p.name}, tmp)')
            return

        if sub.can_inline():
            exp = sub.fields[0].value(sub.header_size)
            w.write(f'{"*" if p.optional else ""}{self.short}.{p.name} = {sub.type_name}({exp})')
        else:
            w.err_check(f'{self.short}.{p.name}.UnmarshalBinary(data[{sub.header_size}:{sub_len}])')

    def alloc(self, w: GoWriter, p: ParamSpec):
        if p.optional and not p.repeatable:
            w.write(f'{self.short}.{p.name} = new({p.p_def.type_name})')

    def len_adv(self, w: GoWriter):
        """Write the code to reslice the data array past this parameter.
        If it's a TLV, subLen must be in scope and equal to it's header's claimed length.
        If it's a TV, it must be fixed size."""
        if self.header_size == 1:
            assert self.fixed_size
            w.write(f'data = data[{self.min_size}:]')
        else:
            w.write(f'data = data[subLen:]')

    def header_len_check(self, w: GoWriter, known_len: int = 0) -> int:
        if self.fixed_size and self.min_size > known_len:
            w.err_check(f'hasEnoughBytes({self.const_name}, {self.min_size}, len(data), false)')
            return self.min_size
        elif self.header_size > known_len:
            w.err_check(f'hasEnoughBytes({self.const_name}, {self.header_size}, len(data), false)')
            return self.header_size
        return known_len

    def sublen_check(self, w: GoWriter) -> bool:
        """Write a check that ensures there's at least as many bytes as the header claims.
        Only applies to TLVs; TVs simply return without writing anything."""

        '''
        if self.fixed_size:
            with w.condition(f'len(data) < {self.min_size}'):
                w.reterr(f'{self.const_name} '
                         f'needs {self.min_size} bytes, but only %d bytes remain',
                         ['len(data)'])
            return False
        '''
        if self.header_size == 1:
            return False
        w.write('subLen := binary.BigEndian.Uint16(data[2:])')
        with w.condition('int(subLen) > len(data)'):
            w.reterr(f'{self.const_name} '
                     'says it has %d bytes, but only %d bytes remain',
                     ['subLen', 'len(data)'])
        return True

    def write_tests(self, w: GoWriter):
        with w.block(f'func Test{self.name}_roundTrip(t *testing.T)'):
            self.write_round_trip_test(w)


@dataclass
class Message(Container):
    """Overrides Container with things that Messages actually do differently than Parameters."""
    response_to: Optional[int] = None  # id of the message this is a response to

    @classmethod
    def from_yaml(cls, y, types: Dict[str, DataType]) -> 'Message':
        y['short'] = y.get('short', 'm')  # change the name of the method receiver
        # noinspection PyTypeChecker
        m: Message = super(Message, cls).from_yaml(y, types)
        return m

    def can_inline(self) -> bool:
        return False  # never inline messages

    @property
    def const_name(self) -> str:
        return self.name

    def len_check(self, w) -> int:
        if self.empty():
            w.comment(f'{self.const_name} is a header-only message')
            with w.condition('len(data) > 0'):
                w.reterr(f'{self.const_name} should be empty, but has %d bytes', ['len(data)'])
            w.write('return nil')
            return 0

        if self.fixed_size:
            with w.condition(f'len(data) != {self.min_size}'):
                w.reterr(f'{self.const_name} should length should be exactly {self.min_size}, '
                         f'but is %d', ['len(data)'])
        else:
            with w.condition(f'len(data) < {self.min_size}'):
                w.reterr(f'{self.const_name} length should be at least {self.min_size}, '
                         f'but is %d', ['len(data)'])
        return self.min_size if self.min_size is not None else 0

    def write_unmarshal(self, w):
        if any(self.fields) or any(self.parameters):
            super().write_unmarshal(w)
            return

        with w.block(f'func ({self.short} *{self.type_name}) UnmarshalBinary(data []byte) error'):
            w.comment(f'{self.name} is a header-only message')
            with w.condition('len(data) > 0'):
                w.reterr(f'{self.name} should be empty, but has %d bytes', ['len(data)'])
            w.write('return nil')

    def write_get_header(self, w):
        return

    def write_marshal_body(self, w):
        w.write('b := bytes.Buffer{}')
        w.err_check(f'{self.short}.EncodeFields(&b)', ret='nil, err')
        for p in self.parameters:
            if p.repeatable:
                with w.block(f'for i := range {self.short}.{p.name}'):
                    w.err_check(f'encodeParams(&b, {self.short}.{p.name}[i].getHeader())', ret='nil, err')
            elif p.optional:
                with w.condition(f'{self.short}.{p.name} != nil'):
                    w.err_check(f'encodeParams(&b, {self.short}.{p.name}.getHeader())', ret='nil, err')
            else:
                w.err_check(f'encodeParams(&b, {self.short}.{p.name}.getHeader())', ret='nil, err')
        w.write('return b.Bytes(), nil')

    def write_encode_body(self, w):
        if any(self.fields) or any(self.parameters):
            super().write_encode_body(w)
            return

        w.comment(f'{self.name} is a header-only message')
        w.write('return nil')

    def write_helpers(self, w: GoWriter):
        """Write the Type() and Status() interface implementations."""
        w.comment(f"Type returns this message's MessageType")
        with w.block(f'func (*{self.type_name}) Type() MessageType'):
            w.write(f'return Msg{self.name}')

        if not any(p.param_name == 'LLRPStatus' for p in self.parameters):
            return

        w.write('')
        w.comment(f"Status returns this message's LLRPStatus")
        with w.block(f'func ({self.short} *{self.type_name}) Status() LLRPStatus'):
            w.write(f'return {self.short}.LLRPStatus')


def load_data(definitions: YML) -> (Dict[str, DataType], Dict[str, Container], Dict[str, Message]):
    """Read the YAML definitions to build the object graph."""

    # predefine some types
    types: Dict[str, DataType] = {
        'bool':     DataType('bool', bits=1),
        'uint8':    DataType('uint8'),
        'byte':     DataType('byte', size=1),
        'uint16':   DataType('uint16'),
        'uint32':   DataType('uint32'),
        'uint64':   DataType('uint64'),
        'int8':     DataType('int8'),
        'int16':    DataType('int16'),
        'int32':    DataType('int32'),
        'int64':    DataType('int64'),
        'string':   DataType(
            name='string', storage='[]byte', size=1,
            description="strings in LLRP are UTF-8 values "
                        "starting with uint16 byte-length header"),
        'bitArray': DataType(
            name='bitArray', storage='[]byte', size=1,
            kind='external', min=0, max=2 ** 16 - 1,
            description="bitArrays in LLRP are a series of bits, "
                        "starting with a uint16 indicating number of bits, "
                        "padded with 0s (as LSBs) to an octet boundary.")
    }

    parameters: Dict[str, Container] = {}

    messages: Dict[str, Message] = {}

    for y in definitions['types']:
        dt = DataType.from_yaml(y, types)
        if dt.name in types:
            raise DuplicateDefError('type', dt.name, dt, types[dt.name], y)
        types[dt.name] = dt

    for y in definitions['parameters']:
        p = Container.from_yaml(y, types)
        if p.name in parameters:
            raise DuplicateDefError(f'parameter', p.name, p, parameters[p.name], y)
        parameters[p.name] = p

    for y in definitions['messages']:
        m = Message.from_yaml(y, types)
        if m.name in messages:
            raise DuplicateDefError(f'message', m.name, m, messages[m.name], y)
        messages[m.name] = m

    return types, parameters, messages


def set_param_sizes(params: Dict[str, Container]):
    """Determine the min sizes of the parameters,
    i.e., the sum of their fields' min sizes
    plus the sum of required parameters' min sizes,
    plus the parameter's header, which varies depending on its type.
    Determine if the parameter's size is in fact fixed."""
    param_sizes: Dict[str, int] = {}  # for handling parameters cycles

    def get_param_size(p: Container) -> int:
        if p.name in param_sizes:
            return param_sizes[p.name]
        param_sizes[p.name] = p.header_size

        p.min_size = (p.header_size +
                      sum(f.min_size for f in p.fields) -
                      sum(f.min_size for f in p.fields if f.partial))
        for (opt, g), p_group in groupby(p.parameters, key=lambda x: (x.optional, x.group)):
            if opt:  # skip optional groups
                for sp in p_group:
                    get_param_size(sp.p_def)
                continue
            p.min_size += min(get_param_size(sp.p_def) for sp in p_group)

        # because of the get_param_size call above, sub-parameter's fixed_size is set
        p.fixed_size = (all(f.is_fixed_size() for f in p.fields) and
                        all(sub.is_fixed_size() for sub in p.parameters))

        param_sizes[p.name] = p.min_size
        return p.min_size

    for p in params.values():
        p.header_size = 4 if p.is_tlv else 1
        for sub in p.parameters:
            try:
                sub.p_def = params[sub.param_name]
            except KeyError:
                raise MissingParamError(p, sub)

    for p in params.values():
        get_param_size(p)
        p.has_required = any(sp for sp in p.parameters if not sp.optional)


def set_msg_sizes(msgs: Dict[str, Message], params: Dict[str, Container]):
    """Set messages min sizes to the sum of their fields' min sizes
    plus the sum of required parameters' min sizes.
    Note that this doesn't include the message header."""

    for m in msgs.values():
        for p in m.parameters:
            try:
                p.p_def = params[p.param_name]
            except KeyError:
                raise MissingParamError(m, p)

        m.min_size = (sum(f.min_size for f in m.fields) -
                      sum(f.min_size for f in m.fields if f.partial) +
                      sum(p.p_def.min_size for p in m.parameters if not p.optional))
        m.fixed_size = (all(f.is_fixed_size() for f in m.fields) and
                        all(sub.is_fixed_size() for sub in m.parameters))
        m.has_required = any(sp.optional for sp in m.parameters)


def write_struct_code(w: GoWriter, types: Dict[str, DataType],
                         parameters: Dict[str, Container], messages: Dict[str, Message],
                         write_header: bool = True):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

    # write types
    for t in types.values():
        t.write_type_def(w)

    with w.paren('const'):
        for m_name, m in messages.items():
            w.write(f'Msg{m_name} = MessageType({m.type_id})')

    w.write('')
    for m in messages.values():
        w.comment(f'{m.type_name} is Message {m.type_id}, {m.name}.')
        if m.description:
            w.write('//')
            w.comment(m.description)

        m.write_struct(w)
        m.write_helpers(w)

    for p in parameters.values():
        if p.type_id == 0:
            continue

        w.comment(f'{p.type_name} is Parameter {p.type_id}, {p.name}.')
        if p.description:
            w.write('//')
            w.comment(p.description)

        p.write_struct(w)


def write_unmarshal_code(w: GoWriter, types: Dict[str, DataType],
                         parameters: Dict[str, Container], messages: Dict[str, Message],
                         write_header: bool = True):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"encoding/binary"')
            w.write('"github.com/pkg/errors"')

    w.comment("hasEnoughBytes returns an error if there aren't "
              "enough bytes to read the parameter.")
    with w.block('func hasEnoughBytes(pt ParamType, needed, got int, exact bool) error'):
        with w.condition('needed <= got'):
            w.write('return nil')
        with w.condition('exact && needed == 0'):
            w.reterr('%v must be empty, but received %d byte(s)', ['pt', 'got'])
            w.ifelse('exact')
            w.reterr('%v requires exactly %d byte(s), but received %d', ['pt', 'needed', 'got'])
        w.reterr('%v requires at least %d byte(s), but received %d', ['pt', 'needed', 'got'])

    w.write('')
    for m in messages.values():
        w.comment(f'UnmarshalBinary Message {m.type_id}, {m.name}.')
        m.write_unmarshal(w)

    for p in parameters.values():
        if p.type_id == 0:
            continue
        w.comment(f'UnmarshalBinary Parameter {p.type_id}, {p.name}.')
        p.write_unmarshal(w)


def write_marshal_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"bytes"')

        w.write('')

    w.comment("b2b converts a bool to a byte, because Go doesn't have a built-in for it.")
    with w.block('func b2b(b bool) byte'):
        with w.condition('b'):
            w.write('return 1')
        w.write('return 0')

    for m in messages.values():
        w.comment(f'MarshalBinary Message {m.type_id}, {m.name}.')
        m.write_marshal(w)

    for p in parameters.values():
        w.comment(f'MarshalBinary Parameter {p.type_id}, {p.name}.')
        p.write_marshal(w)


def write_encode_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            w.write('"io"')
            w.write('"encoding/binary"')
            w.write('"github.com/pkg/errors"')

    for m in messages.values():
        w.comment(f'EncodeFields for Message {m.type_id}, {m.name}.')
        m.write_encode(w)

    for p in parameters.values():
        w.comment(f'EncodeFields for Parameter {p.type_id}, {p.name}.')
        p.write_encode(w)


def write_test_code(w, types, parameters, messages, write_header=False):
    if write_header:
        w.comment(f'Code generated by "{" ".join(sys.argv)}"; DO NOT EDIT.\n', auto_break=False)
        w.write('\npackage llrp\n\n')

        with w.paren('import'):
            # w.write('"encoding/binary"')
            w.write('"testing"')
            w.write('"reflect"')
            # w.write('"github.com/pkg/errors"')

        w.write('')

    for m in messages.values():
        w.comment(f'Test Message {m.type_id}, {m.name}.')
        m.write_tests(w)

    for p in parameters.values():
        w.comment(f'Test Parameter {p.type_id}, {p.name}.')
        p.write_tests(w)


def main():
    import sys
    import argparse
    import subprocess

    parser = argparse.ArgumentParser(
        description='This is a python script to read a YAML description '
                    'of binary LLRP messages to generate Go code '
                    'that converts them to and from JSON.'
    )
    parser.add_argument('-i', '--input', help='input YAML file (default: STDIN)',
                        type=argparse.FileType('r'), default=sys.stdin)

    def add_out_file(kind: str):
        parser.add_argument(
            f'-{kind[0]}',
            f'--{kind}-file',
            default='-',
            help=f'output file for {kind} code (default: STDOUT)')

    # output files; first letter is used as argument
    add_out_file('structs')
    add_out_file('unmarshal')
    add_out_file('marshal')
    add_out_file('encode')
    add_out_file('test')

    def add_flag(name: str, help: str):
        parser.add_argument(
            f'--{name}', default=True, action='store_true',
            help=f'{help} (default true)')
        parser.add_argument(
            f'--no-{name}', dest=name, action='store_false',
            help=f"don't {help}")

    # argument names + descriptions for operations
    add_flag('structs', 'write Go structs and types')
    add_flag('unmarshal', 'write LLRP binary -> Go struct unmarshaling code')
    add_flag('marshal', 'write Go struct -> LLRP binary marshaling code')
    add_flag('encode', 'write Go -> LLRP binary fields encoder code')
    add_flag('test', 'write test code')
    add_flag('gofmt', 'pipe output through gofmt')

    parser.add_argument(
        '--parameter',
        help='output only specific parameters (repeatable)',
        action='append')

    args = parser.parse_args()

    definitions = yaml.safe_load(args.input)
    types, parameters, messages = load_data(definitions)
    set_param_sizes(parameters)
    set_msg_sizes(messages, parameters)

    @contextmanager
    def gowrite(filename: str, gofmt: bool):
        with ExitStack() as stack:
            out_file = sys.stdout
            if filename != '-':
                out_file = stack.enter_context(open(filename, 'w'))

            if gofmt:
                gofmt_proc = stack.enter_context(subprocess.Popen(
                    ['gofmt'],
                    stdin=subprocess.PIPE,
                    stdout=out_file,
                    stderr=subprocess.STDOUT,
                    encoding='utf-8'))
                out_file = gofmt_proc.stdin

            yield GoWriter(out_file)

    stdout_has_header = False
    # iterates over (argument variable, argument file, func to run if arg is true,
    # func to run if only for specific params)
    for arg, f, func, p_func in [
        (args.structs, args.structs_file, write_struct_code, Container.write_struct),
        (args.unmarshal, args.unmarshal_file, write_unmarshal_code, Container.write_unmarshal),
        (args.marshal, args.marshal_file, write_marshal_code, Container.write_marshal),
        (args.encode, args.encode_file, write_encode_code, Container.write_encode),
        (args.test, args.test_file, write_test_code, Container.write_tests),
    ]:
        if not arg:
            continue

        with gowrite(f, args.gofmt) as w:
            write_header = True
            if f == '-':
                write_header = stdout_has_header
                stdout_has_header = True

            if args.parameter:
                for p_name in args.parameter:
                    p_func(args.parameter[p_name], w)
            else:
                func(w, types, parameters, messages, write_header)
            w.write('\n')


if __name__ == '__main__':
    try:
        main()
    except DefinitionError as e:
        print(e.yml)
        raise
    except MissingParamError as e:
        print(e.parent)
        print(e.decl)
        raise

# #  Copyright (C) 2020 Intel Corporation # #  SPDX-License-Identifier: Apache-2.0 # This python script reads a YAML file # to generate Go code that unpacks binary LLRP messages # so that they can be marshaled into JSON. # # Why bother doing it this way? In short, because other ways weren't good; # it's much easier to express the LLRP message hierarchy as YAML than as Go code, # and it's pretty easy to use that to generate parsers that are far less error prone # than writing it by hand. # # At first blush, LLRP looks easy to parse, # but when you dig in, there are a lot of rules and required context # around how the next bytes should be interpreted. # I tried writing it with interfaces and with reflection code, # but it was way, way too easy to make a mistake, and I spent a ton of time # debugging difficult to spot errors that could only be triggered at runtime. # # The best version came from 'hand-crafted' byte unmarshaling, # but 1) it's still easy to mess up and 2) it would take too long to write. # # As I was building up a mental model around LLRP, # I built a spreadsheet to reference the memory layouts, types, and rules. # Looking at it, I realized it had all the need parsing info, # so I converted to it the messages.yaml file and cleaned it up a bit, # then wrote this code to read it in, statically check various assumptions, # calculate some useful constraint checks from the message graph, # and output the unmarshaling code. # just for type checking; otherwise does nothing Base class for errors generated by this script. Message or Parameter references an unknown Parameter. Problem with the YAML definition. GoWriter outputs (mostly) formatted Go code.     By default it writes to stdout, but you can set w to any File object     (really, anything with a 'write' function). # map of basic types to byte lengths Convert ABCCamelCaseString to abcCamelCaseString.         Useful for creating unexported type names. Increase the indentation level for all subsequent output. Decrease the indentation level fro all subsequent output. Returns a context within which writes are indented inside braces,         defined by the style. By default, this looks like:              {                 <output written within context>             }             <next write goes here>          Use like any other python context. E.g., this code:             with writer.block('const', style='()'):                 for i in range(3):                     w.write(f'myNum{i} = 1<<{i}')         has this output:             const (                 myNum0 = 1<<0                 myNum1 = 1<<1                 myNum2 = 1<<2             ) Returns a context within which writes are indented inside parentheses. Returns a context within which writes are indented inside an if (cond) block. Write the '} else {' or '} else if (condition) {' in an if...else block. Returns a context for writing switch cases inside curly braces.         See paren for more information. Write 'if err := (cond); err != nil {\n return err\n }.         Optionally, set other parameters and/or change the return value:             if params[0], params[1], err := cond; err != nil {\n return ret\n} Writes code to return an error, using 'return err' or one of         New, Errorf, Wrap, or Wrapf if there are params and/or wrap is True. DataType backs field types, such as built-ins, flags, enumerations, and aliases. # underlying basic type (e.g., uint8) # type name must be unique to the Go package # when generating code, prefix is attached to enum/flag names # flag, enumeration, or alias, or external # maps name -> value, or implicitly name -> iota # storage byte size; for arrays, the byte size of a single element # number of bits in the final byte; usually should be 8, but occasionally not # for outputting doc comments # used for testing; set to reasonable values if not specified in YAML # allow the YAML value to use a list to map name -> position False for array types and True for all others. Returns valid Go code for an instance of the correct type,         possibly using i as a seed.         For a given positive value of i, this always returns the same value,         but different values of i won't necessarily result in different values. A field spec defines a field in a parameter or message. # the name to use in structs # backing type; for arrays, the type of a single element # 0 if has a length prefix, -1 if all remaining bytes, otherwise an array length # for doc comments # currently unused; eventually might be used for validation # currently unused # if the next field shares its bits with this one # starting bit position, where 0=MSB ("leftmost") # test values - set to sensible values after parsing YAML if not defined explicitly # num items in generated arrays (if not fixed-size) Returns the minimum number of bytes this field needs in an LLRP message. Returns True if this field has a fixed size. Returns the variable name used to reference this field. Returns the field type as it should appear in a struct definition. Returns a string for a new instance of this type, for use in tests. Return an expression to extract data[begin:end] and cast to this type. # determine if we need to cast Writes unmarshaling code for an array type.         yields an expression to reslice data past what it read. # for fixed size arrays # bit arrays require special length handling # strings implicitly copy bytes during type conversion from []byte # all other array types # used in error strings Outputs Go code to unmarshal this fields.         If is_last is False and this is not a fixed size type,         it'll writes an expression to reslice the data. # consume the rest of the data # TODO: handle overflow checking ParamSpec specifies the parameters permitted in a certain context,     and under what conditions. # name used for a struct; may be omitted from yaml to use the parameter name # name of an LLRP parameter type; called 'type' in the YAML # parameter is "0-1" or "0-n" # parameter is "0-n" or "1-n" # if this parameter is tied to a specific air protocol... # parameter definition; set after parameters are read # groups are mutually exclusive Returns true if this parameter is always a fixed size. Container mostly represents Parameters, but Messages are nearly identical,     so it serves double duty. # message or parameter number # string to use as method receivers # derived properties # true if no variable length fields/optional or repeatable parameters # minimum required size (incl. header); absolute size if fixed_size is True # only set for Parameters, not Messages # true if has required parameters Returns the Go type name. # return GoWriter.lower_camel(self.name) Returns True if this doesn't have any fields or parameters. Returns False if this must be a struct. Write the Parameter as a Go struct. Returns True if after unmarshaling, we should check for remaining data. Write a 'name := value' expression that assigns a new test instance of this type. # write a struct literal # 02x\\n%+v\\n%+v", err, b, {self.short}, {self.short}2)') # 02x\\n%+v\\n%+v", b, {self.short}, {self.short}2)') # all remaining data, so no length header # byte length check # field unmarshaling # skip reslice if this is a fixed size field, # or if it's the final item (i.e., last field & no params) # parameters/sub-parameters # w.comment(f'known data length: {known_data_len}') # simple path: there's only a single parameter, or all in p_group are required # otherwise, the next parameter is one of a collection of # repeatable, optional, intermixed, or mutually-exclusive parameters # special weirdness: the next param could be a single byte TV # or it could be a TLV with a 4 byte header, # so we have to check how much data is available # w.write(f'tmp := new({sub.type_name})') Write the code to reslice the data array past this parameter.         If it's a TLV, subLen must be in scope and equal to it's header's claimed length.         If it's a TV, it must be fixed size. Write a check that ensures there's at least as many bytes as the header claims.         Only applies to TLVs; TVs simply return without writing anything. if self.fixed_size:             with w.condition(f'len(data) < {self.min_size}'):                 w.reterr(f'{self.const_name} '                          f'needs {self.min_size} bytes, but only %d bytes remain',                          ['len(data)'])             return False Overrides Container with things that Messages actually do differently than Parameters. # id of the message this is a response to # change the name of the method receiver # noinspection PyTypeChecker # never inline messages Write the Type() and Status() interface implementations. Read the YAML definitions to build the object graph. # predefine some types Determine the min sizes of the parameters,     i.e., the sum of their fields' min sizes     plus the sum of required parameters' min sizes,     plus the parameter's header, which varies depending on its type.     Determine if the parameter's size is in fact fixed. # for handling parameters cycles # skip optional groups # because of the get_param_size call above, sub-parameter's fixed_size is set Set messages min sizes to the sum of their fields' min sizes     plus the sum of required parameters' min sizes.     Note that this doesn't include the message header. # write types # w.write('"encoding/binary"') # w.write('"github.com/pkg/errors"') # output files; first letter is used as argument # argument names + descriptions for operations # iterates over (argument variable, argument file, func to run if arg is true, # func to run if only for specific params)