# -*- coding: utf-8 -*-
from __future__ import print_function
"""
Copyright (c) 2013, Mahmoud Hashemi

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.

    * Redistributions in binary form must reproduce the above
      copyright notice, this list of conditions and the following
      disclaimer in the documentation and/or other materials provided
      with the distribution.

    * The names of the contributors may not be used to endorse or
      promote products derived from this software without specific
      prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

"""
Forked from https://github.com/mahmoud/boltons/pull/217
"""

"""Python's built-in :mod:`functools` module builds several useful
utilities on top of Python's first-class function
support. ``funcutils`` generally stays in the same vein, adding to and
correcting Python's standard metaprogramming facilities.
"""
# flake8: noqa
import sys
import re
import inspect
import functools
import itertools
from types import MethodType, FunctionType

try:
    xrange
    make_method = MethodType
except NameError:
    # Python 3
    make_method = lambda desc, obj, obj_type: MethodType(desc, obj)  # noqa
    basestring = (str, bytes)  # Python 3 compat
    _IS_PY2 = False
else:
    _IS_PY2 = True

try:
    _inspect_iscoroutinefunction = inspect.iscoroutinefunction
except AttributeError:
    # Python 3.4
    _inspect_iscoroutinefunction = lambda func: False  # noqa

try:
    from boltons.typeutils import make_sentinel

    NO_DEFAULT = make_sentinel(var_name="NO_DEFAULT")
except ImportError:
    NO_DEFAULT = object()

def get_module_callables(mod, ignore=None):
    """Returns two maps of (*types*, *funcs*) from *mod*, optionally
    ignoring based on the :class:`bool` return value of the *ignore*
    callable. *mod* can be a string name of a module in
    :data:`sys.modules` or the module instance itself.
    """
    if isinstance(mod, basestring):
        mod = sys.modules[mod]
    types, funcs = {}, {}
    for attr_name in dir(mod):
        if ignore and ignore(attr_name):
            continue
        try:
            attr = getattr(mod, attr_name)
        except Exception:
            continue
        try:
            attr_mod_name = attr.__module__
        except AttributeError:
            continue
        if attr_mod_name != mod.__name__:
            continue
        if isinstance(attr, type):
            types[attr_name] = attr
        elif callable(attr):
            funcs[attr_name] = attr
    return types, funcs

def mro_items(type_obj):
    """Takes a type and returns an iterator over all class variables
    throughout the type hierarchy (respecting the MRO).

    >>> sorted(set([k for k, v in mro_items(int) if not k.startswith('__') and 'bytes' not in k and not callable(v)]))
    ['denominator', 'imag', 'numerator', 'real']
    """
    # TODO: handle slots?
    return itertools.chain.from_iterable(ct.__dict__.items() for ct in type_obj.__mro__)

def dir_dict(obj, raise_exc=False):
    """Return a dictionary of attribute names to values for a given
    object. Unlike ``obj.__dict__``, this function returns all
    attributes on the object, including ones on parent classes.
    """
    # TODO: separate function for handling descriptors on types?
    ret = {}
    for k in dir(obj):
        try:
            ret[k] = getattr(obj, k)
        except Exception:
            if raise_exc:
                raise
    return ret

def copy_function(orig, copy_dict=True):
    """Returns a shallow copy of the function, including code object,
    globals, closure, etc.

    >>> func = lambda: func
    >>> func() is func
    True
    >>> func_copy = copy_function(func)
    >>> func_copy() is func
    True
    >>> func_copy is not func
    True

    Args:
        orig (function): The function to be copied. Must be a
            function, not just any method or callable.
        copy_dict (bool): Also copy any attributes set on the function
            instance. Defaults to ``True``.
    """
    ret = FunctionType(
        orig.__code__,
        orig.__globals__,
        name=orig.__name__,
        argdefs=getattr(orig, "__defaults__", None),
        closure=getattr(orig, "__closure__", None),
    )
    if copy_dict:
        ret.__dict__.update(orig.__dict__)
    return ret

def partial_ordering(cls):
    """Class decorator, similar to :func:`functools.total_ordering`,
    except it is used to define `partial orderings`_ (i.e., it is
    possible that *x* is neither greater than, equal to, or less than
    *y*). It assumes the presence of the ``__le__()`` and ``__ge__()``
    method, but nothing else. It will not override any existing
    additional comparison methods.

    .. _partial orderings: https://en.wikipedia.org/wiki/Partially_ordered_set

    >>> @partial_ordering
    ... class MySet(set):
    ...     def __le__(self, other):
    ...         return self.issubset(other)
    ...     def __ge__(self, other):
    ...         return self.issuperset(other)
    ...
    >>> a = MySet([1,2,3])
    >>> b = MySet([1,2])
    >>> c = MySet([1,2,4])
    >>> b < a
    True
    >>> b > a
    False
    >>> b < c
    True
    >>> a < c
    False
    >>> c > a
    False
    """

    def __lt__(self, other):
        return self <= other and not self >= other

    def __gt__(self, other):
        return self >= other and not self <= other

    def __eq__(self, other):
        return self >= other and self <= other

    if not hasattr(cls, "__lt__"):
        cls.__lt__ = __lt__
    if not hasattr(cls, "__gt__"):
        cls.__gt__ = __gt__
    if not hasattr(cls, "__eq__"):
        cls.__eq__ = __eq__

    return cls

class InstancePartial(functools.partial):
    """:class:`functools.partial` is a huge convenience for anyone
    working with Python's great first-class functions. It allows
    developers to curry arguments and incrementally create simpler
    callables for a variety of use cases.

    Unfortunately there's one big gap in its usefulness:
    methods. Partials just don't get bound as methods and
    automatically handed a reference to ``self``. The
    ``InstancePartial`` type remedies this by inheriting from
    :class:`functools.partial` and implementing the necessary
    descriptor protocol. There are no other differences in
    implementation or usage. :class:`CachedInstancePartial`, below,
    has the same ability, but is slightly more efficient.

    """

    def __get__(self, obj, obj_type):
        return make_method(self, obj, obj_type)

class CachedInstancePartial(functools.partial):
    """The ``CachedInstancePartial`` is virtually the same as
    :class:`InstancePartial`, adding support for method-usage to
    :class:`functools.partial`, except that upon first access, it
    caches the bound method on the associated object, speeding it up
    for future accesses, and bringing the method call overhead to
    about the same as non-``partial`` methods.

    See the :class:`InstancePartial` docstring for more details.
    """

    def __get__(self, obj, obj_type):
        # These assignments could've been in __init__, but there was
        # no simple way to do it without breaking one of PyPy or Py3.
        self.__name__ = None
        self.__doc__ = self.func.__doc__
        self.__module__ = self.func.__module__

        name = self.__name__
        if name is None:
            for k, v in mro_items(obj_type):
                if v is self:
                    self.__name__ = name = k
        if obj is None:
            return make_method(self, obj, obj_type)
        try:
            # since this is a data descriptor, this block
            # is probably only hit once (per object)
            return obj.__dict__[name]
        except KeyError:
            obj.__dict__[name] = ret = make_method(self, obj, obj_type)
            return ret

partial = CachedInstancePartial

# # #
# # # Function builder
# # #

def wraps(func, injected=None, expected=None, **kw):
    """Modeled after the built-in :func:`functools.wraps`, this function is
    used to make your decorator's wrapper functions reflect the
    wrapped function's:

      * Name
      * Documentation
      * Module
      * Signature

    The built-in :func:`functools.wraps` copies the first three, but
    does not copy the signature. This version of ``wraps`` can copy
    the inner function's signature exactly, allowing seamless usage
    and :mod:`introspection <inspect>`. Usage is identical to the
    built-in version::

        >>> from boltons.funcutils import wraps
        >>>
        >>> def print_return(func):
        ...     @wraps(func)
        ...     def wrapper(*args, **kwargs):
        ...         ret = func(*args, **kwargs)
        ...         print(ret)
        ...         return ret
        ...     return wrapper
        ...
        >>> @print_return
        ... def example():
        ...     '''docstring'''
        ...     return 'example return value'
        >>>
        >>> val = example()
        example return value
        >>> example.__name__
        'example'
        >>> example.__doc__
        'docstring'

    In addition, the boltons version of wraps supports modifying the
    outer signature based on the inner signature. By passing a list of
    *injected* argument names, those arguments will be removed from
    the outer wrapper's signature, allowing your decorator to provide
    arguments that aren't passed in.

    Args:

        func (function): The callable whose attributes are to be copied.
        injected (list): An optional list of argument names which
            should not appear in the new wrapper's signature.
        expected (list): An optional list of argument names (or (name,
            default) pairs) representing new arguments introduced by
            the wrapper (the opposite of *injected*). See
            :meth:`FunctionBuilder.add_arg()` for more details.
        update_dict (bool): Whether to copy other, non-standard
            attributes of *func* over to the wrapper. Defaults to True.
        inject_to_varkw (bool): Ignore missing arguments when a
            ``**kwargs``-type catch-all is present. Defaults to True.

    For more in-depth wrapping of functions, see the
    :class:`FunctionBuilder` type, on which wraps was built.

    """
    if injected is None:
        injected = []
    elif isinstance(injected, basestring):
        injected = [injected]
    else:
        injected = list(injected)

    expected_items = _parse_wraps_expected(expected)

    if isinstance(func, (classmethod, staticmethod)):
        raise TypeError(
            "wraps does not support wrapping classmethods and"
            " staticmethods, change the order of wrapping to"
            " wrap the underlying function: %r" % (getattr(func, "__func__", None),)
        )

    update_dict = kw.pop("update_dict", True)
    inject_to_varkw = kw.pop("inject_to_varkw", True)
    if kw:
        raise TypeError("unexpected kwargs: %r" % kw.keys())

    fb = FunctionBuilder.from_func(func)
    for arg in injected:
        try:
            fb.remove_arg(arg)
        except MissingArgument:
            if inject_to_varkw and fb.varkw is not None:
                continue  # keyword arg will be caught by the varkw
            raise

    for arg, default in expected_items:
        fb.add_arg(arg, default)  # may raise ExistingArgument

    if fb.is_async:
        fb.body = "return await _call(%s)" % fb.get_invocation_str()
    else:
        fb.body = "return _call(%s)" % fb.get_invocation_str()

    def wrapper_wrapper(wrapper_func):
        execdict = dict(_call=wrapper_func, _func=func)
        fully_wrapped = fb.get_func(execdict, with_dict=update_dict)
        fully_wrapped.__wrapped__ = func  # ref to the original function (#115)

        return fully_wrapped

    return wrapper_wrapper

def _parse_wraps_expected(expected):
    # expected takes a pretty powerful argument, it's processed
    # here. admittedly this would be less trouble if I relied on
    # OrderedDict (there's an impl of that in the commit history if
    # you look
    if expected is None:
        expected = []
    elif isinstance(expected, basestring):
        expected = [(expected, NO_DEFAULT)]

    expected_items = []
    try:
        expected_iter = iter(expected)
    except TypeError as e:
        raise ValueError(
            '"expected" takes string name, sequence of string names,'
            " iterable of (name, default) pairs, or a mapping of "
            " {name: default}, not %r (got: %r)" % (expected, e)
        )
    for argname in expected_iter:
        if isinstance(argname, basestring):
            # dict keys and bare strings
            try:
                default = expected[argname]
            except TypeError:
                default = NO_DEFAULT
        else:
            # pairs
            try:
                argname, default = argname
            except (TypeError, ValueError):
                raise ValueError(
                    '"expected" takes string name, sequence of string names,'
                    " iterable of (name, default) pairs, or a mapping of "
                    " {name: default}, not %r"
                )
        if not isinstance(argname, basestring):
            raise ValueError(
                'all "expected" argnames must be strings, not %r' % (argname,)
            )

        expected_items.append((argname, default))

    return expected_items

class FunctionBuilder(object):
    """The FunctionBuilder type provides an interface for programmatically
    creating new functions, either based on existing functions or from
    scratch.

    Values are passed in at construction or set as attributes on the
    instance. For creating a new function based of an existing one,
    see the :meth:`~FunctionBuilder.from_func` classmethod. At any
    point, :meth:`~FunctionBuilder.get_func` can be called to get a
    newly compiled function, based on the values configured.

    >>> fb = FunctionBuilder('return_five', doc='returns the integer 5',
    ...                      body='return 5')
    >>> f = fb.get_func()
    >>> f()
    5
    >>> fb.varkw = 'kw'
    >>> f_kw = fb.get_func()
    >>> f_kw(ignored_arg='ignored_val')
    5

    Note that function signatures themselves changed quite a bit in
    Python 3, so several arguments are only applicable to
    FunctionBuilder in Python 3. Except for *name*, all arguments to
    the constructor are keyword arguments.

    Args:
        name (str): Name of the function.
        doc (str): `Docstring`_ for the function, defaults to empty.
        module (str): Name of the module from which this function was
            imported. Defaults to None.
        body (str): String version of the code representing the body
            of the function. Defaults to ``'pass'``, which will result
            in a function which does nothing and returns ``None``.
        args (list): List of argument names, defaults to empty list,
            denoting no arguments.
        varargs (str): Name of the catch-all variable for positional
            arguments. E.g., "args" if the resultant function is to have
            ``*args`` in the signature. Defaults to None.
        varkw (str): Name of the catch-all variable for keyword
            arguments. E.g., "kwargs" if the resultant function is to have
            ``**kwargs`` in the signature. Defaults to None.
        defaults (dict): A mapping of argument names to default values.
        kwonlyargs (list): Argument names which are only valid as
            keyword arguments. **Python 3 only.**
        kwonlydefaults (dict): A mapping, same as normal *defaults*,
            but only for the *kwonlyargs*. **Python 3 only.**
        annotations (dict): Mapping of type hints and so
            forth. **Python 3 only.**
        filename (str): The filename that will appear in
            tracebacks. Defaults to "boltons.funcutils.FunctionBuilder".
        indent (int): Number of spaces with which to indent the
            function *body*. Values less than 1 will result in an error.
        dict (dict): Any other attributes which should be added to the
            functions compiled with this FunctionBuilder.

    All of these arguments are also made available as attributes which
    can be mutated as necessary.

    .. _Docstring: https://en.wikipedia.org/wiki/Docstring#Python

    """

    if _IS_PY2:
        _argspec_defaults = {
            "args": list,
            "varargs": lambda: None,
            "varkw": lambda: None,
            "defaults": lambda: None,
        }

        @classmethod
        def _argspec_to_dict(cls, f):
            args, varargs, varkw, defaults = inspect.getargspec(f)
            return {
                "args": args,
                "varargs": varargs,
                "varkw": varkw,
                "defaults": defaults,
            }

    else:
        _argspec_defaults = {
            "args": list,
            "varargs": lambda: None,
            "varkw": lambda: None,
            "defaults": lambda: None,
            "kwonlyargs": list,
            "kwonlydefaults": dict,
            "annotations": dict,
        }

        @classmethod
        def _argspec_to_dict(cls, f):
            argspec = inspect.getfullargspec(f)
            return dict(
                (attr, getattr(argspec, attr)) for attr in cls._argspec_defaults
            )

    _defaults = {
        "doc": str,
        "dict": dict,
        "is_async": lambda: False,
        "module": lambda: None,
        "body": lambda: "pass",
        "indent": lambda: 4,
        "annotations": dict,
        "filename": lambda: "boltons.funcutils.FunctionBuilder",
    }

    _defaults.update(_argspec_defaults)

    _compile_count = itertools.count()

    def __init__(self, name, **kw):
        self.name = name
        for a, default_factory in self._defaults.items():
            val = kw.pop(a, None)
            if val is None:
                val = default_factory()
            setattr(self, a, val)

        if kw:
            raise TypeError("unexpected kwargs: %r" % kw.keys())
        return

    # def get_argspec(self):  # TODO

    if _IS_PY2:

        def get_sig_str(self, with_annotations=True):
            """Return function signature as a string.

            with_annotations is ignored on Python 2.  On Python 3 signature
            will omit annotations if it is set to False.
            """
            return inspect.formatargspec(self.args, self.varargs, self.varkw, [])

        def get_invocation_str(self):
            return inspect.formatargspec(self.args, self.varargs, self.varkw, [])[1:-1]

    else:

        def get_sig_str(self, with_annotations=True):
            """Return function signature as a string.

            with_annotations is ignored on Python 2.  On Python 3 signature
            will omit annotations if it is set to False.
            """
            if with_annotations:
                annotations = self.annotations
            else:
                annotations = {}
            return formatargspec(
                self.args,
                self.varargs,
                self.varkw,
                [],
                self.kwonlyargs,
                {},
                annotations,
            )

        _KWONLY_MARKER = re.compile(
            r"""
        \*     # a star
        \s*    # followed by any amount of whitespace
        ,      # followed by a comma
        \s*    # followed by any amount of whitespace
        """,
            re.VERBOSE,
        )

        def get_invocation_str(self):
            kwonly_pairs = None
            formatters = {}
            if self.kwonlyargs:
                kwonly_pairs = dict((arg, arg) for arg in self.kwonlyargs)
                formatters["formatvalue"] = lambda value: "=" + value

            sig = formatargspec(
                self.args,
                self.varargs,
                self.varkw,
                [],
                kwonly_pairs,
                kwonly_pairs,
                {},
                **formatters
            )
            sig = self._KWONLY_MARKER.sub("", sig)
            return sig[1:-1]

    @classmethod
    def from_func(cls, func):
        """Create a new FunctionBuilder instance based on an existing
        function. The original function will not be stored or
        modified.
        """
        # TODO: copy_body? gonna need a good signature regex.
        # TODO: might worry about __closure__?
        if not callable(func):
            raise TypeError("expected callable object, not %r" % (func,))

        kwargs = {
            "name": func.__name__,
            "doc": func.__doc__,
            "module": func.__module__,
            "annotations": getattr(func, "__annotations__", {}),
            "dict": getattr(func, "__dict__", {}),
        }

        kwargs.update(cls._argspec_to_dict(func))

        if _inspect_iscoroutinefunction(func):
            kwargs["is_async"] = True

        return cls(**kwargs)

    def get_func(self, execdict=None, add_source=True, with_dict=True):
        """Compile and return a new function based on the current values of
        the FunctionBuilder.

        Args:
            execdict (dict): The dictionary representing the scope in
                which the compilation should take place. Defaults to an empty
                dict.
            add_source (bool): Whether to add the source used to a
                special ``__source__`` attribute on the resulting
                function. Defaults to True.
            with_dict (bool): Add any custom attributes, if
                applicable. Defaults to True.

        To see an example of usage, see the implementation of
        :func:`~boltons.funcutils.wraps`.
        """
        execdict = execdict or {}
        body = self.body or self._default_body

        tmpl = "def {name}{sig_str}:"
        tmpl += "\n{body}"

        if self.is_async:
            tmpl = "async " + tmpl

        body = _indent(self.body, " " * self.indent)

        name = self.name.replace("<", "_").replace(">", "_")  # lambdas
        src = tmpl.format(
            name=name,
            sig_str=self.get_sig_str(with_annotations=False),
            doc=self.doc,
            body=body,
        )
        self._compile(src, execdict)
        func = execdict[name]

        func.__name__ = self.name
        func.__doc__ = self.doc
        func.__defaults__ = self.defaults
        if not _IS_PY2:
            func.__kwdefaults__ = self.kwonlydefaults
            func.__annotations__ = self.annotations

        if with_dict:
            func.__dict__.update(self.dict)
        func.__module__ = self.module
        # TODO: caller module fallback?

        if add_source:
            func.__source__ = src

        return func

    def get_defaults_dict(self):
        """Get a dictionary of function arguments with defaults and the
        respective values.
        """
        ret = dict(
            reversed(list(zip(reversed(self.args), reversed(self.defaults or []))))
        )
        kwonlydefaults = getattr(self, "kwonlydefaults", None)
        if kwonlydefaults:
            ret.update(kwonlydefaults)
        return ret

    def get_arg_names(self, only_required=False):
        arg_names = tuple(self.args) + tuple(getattr(self, "kwonlyargs", ()))
        if only_required:
            defaults_dict = self.get_defaults_dict()
            arg_names = tuple([an for an in arg_names if an not in defaults_dict])
        return arg_names

    if _IS_PY2:

        def add_arg(self, arg_name, default=NO_DEFAULT):
            "Add an argument with optional *default* (defaults to ``funcutils.NO_DEFAULT``)."
            if arg_name in self.args:
                raise ExistingArgument(
                    "arg %r already in func %s arg list" % (arg_name, self.name)
                )
            self.args.append(arg_name)
            if default is not NO_DEFAULT:
                self.defaults = (self.defaults or ()) + (default,)
            return

    else:

        def add_arg(self, arg_name, default=NO_DEFAULT, kwonly=False):
            """Add an argument with optional *default* (defaults to
            ``funcutils.NO_DEFAULT``). Pass *kwonly=True* to add a
            keyword-only argument
            """
            if arg_name in self.args:
                raise ExistingArgument(
                    "arg %r already in func %s arg list" % (arg_name, self.name)
                )
            if arg_name in self.kwonlyargs:
                raise ExistingArgument(
                    "arg %r already in func %s kwonly arg list" % (arg_name, self.name)
                )
            if not kwonly:
                self.args.append(arg_name)
                if default is not NO_DEFAULT:
                    self.defaults = (self.defaults or ()) + (default,)
            else:
                self.kwonlyargs.append(arg_name)
                if default is not NO_DEFAULT:
                    self.kwonlydefaults[arg_name] = default
            return

    def remove_arg(self, arg_name):
        """Remove an argument from this FunctionBuilder's argument list. The
        resulting function will have one less argument per call to
        this function.

        Args:
            arg_name (str): The name of the argument to remove.

        Raises a :exc:`ValueError` if the argument is not present.

        """
        args = self.args
        d_dict = self.get_defaults_dict()
        try:
            args.remove(arg_name)
        except ValueError:
            try:
                self.kwonlyargs.remove(arg_name)
            except (AttributeError, ValueError):
                # py2, or py3 and missing from both
                exc = MissingArgument(
                    "arg %r not found in %s argument list:"
                    " %r" % (arg_name, self.name, args)
                )
                exc.arg_name = arg_name
                raise exc
            else:
                self.kwonlydefaults.pop(arg_name, None)
        else:
            d_dict.pop(arg_name, None)
            self.defaults = tuple([d_dict[a] for a in args if a in d_dict])
        return

    def _compile(self, src, execdict):

        filename = "<%s-%d>" % (self.filename, next(self._compile_count))
        try:
            code = compile(src, filename, "single")
            exec(code, execdict)
        except Exception:
            raise
        return execdict

class MissingArgument(ValueError):
    pass

class ExistingArgument(ValueError):
    pass

def _indent(text, margin, newline="\n", key=bool):
    "based on boltons.strutils.indent"
    indented_lines = [
        (margin + line if key(line) else line) for line in text.splitlines()
    ]
    return newline.join(indented_lines)

try:
    from functools import total_ordering  # 2.7+
except ImportError:
    # python 2.6
    def total_ordering(cls):
        """Class decorator that fills in missing comparators/ordering
        methods. Backport of :func:`functools.total_ordering` to work
        with Python 2.6.

        Code from http://code.activestate.com/recipes/576685/
        """
        convert = {
            "__lt__": [
                ("__gt__", lambda self, other: not (self < other or self == other)),
                ("__le__", lambda self, other: self < other or self == other),
                ("__ge__", lambda self, other: not self < other),
            ],
            "__le__": [
                ("__ge__", lambda self, other: not self <= other or self == other),
                ("__lt__", lambda self, other: self <= other and not self == other),
                ("__gt__", lambda self, other: not self <= other),
            ],
            "__gt__": [
                ("__lt__", lambda self, other: not (self > other or self == other)),
                ("__ge__", lambda self, other: self > other or self == other),
                ("__le__", lambda self, other: not self > other),
            ],
            "__ge__": [
                ("__le__", lambda self, other: (not self >= other) or self == other),
                ("__gt__", lambda self, other: self >= other and not self == other),
                ("__lt__", lambda self, other: not self >= other),
            ],
        }
        roots = set(dir(cls)) & set(convert)
        if not roots:
            raise ValueError(
                "must define at least one ordering operation:" " < > <= >="
            )
        root = max(roots)  # prefer __lt__ to __le__ to __gt__ to __ge__
        for opname, opfunc in convert[root]:
            if opname not in roots:
                opfunc.__name__ = opname
                opfunc.__doc__ = getattr(int, opname).__doc__
                setattr(cls, opname, opfunc)
        return cls

if not _IS_PY2:
    # copied from python 3 implementation without deprecation warning
    def formatargspec(
        args,
        varargs=None,
        varkw=None,
        defaults=None,
        kwonlyargs=(),
        kwonlydefaults={},
        annotations={},
        formatarg=str,
        formatvarargs=lambda name: "*" + name,
        formatvarkw=lambda name: "**" + name,
        formatvalue=lambda value: "=" + repr(value),
        formatreturns=lambda text: " -> " + text,
        formatannotation=inspect.formatannotation,
    ):
        """Format an argument spec from the values returned by getfullargspec.

        The first seven arguments are (args, varargs, varkw, defaults,
        kwonlyargs, kwonlydefaults, annotations).  The other five arguments
        are the corresponding optional formatting functions that are called to
        turn names and values into strings.  The last argument is an optional
        function to format the sequence of arguments.

        Deprecated since Python 3.5: use the `signature` function and `Signature`
        objects.
        """

        def formatargandannotation(arg):
            result = formatarg(arg)
            if arg in annotations:
                result += ": " + formatannotation(annotations[arg])
            return result

        specs = []
        if defaults:
            firstdefault = len(args) - len(defaults)
        for i, arg in enumerate(args):
            spec = formatargandannotation(arg)
            if defaults and i >= firstdefault:
                spec = spec + formatvalue(defaults[i - firstdefault])
            specs.append(spec)
        if varargs is not None:
            specs.append(formatvarargs(formatargandannotation(varargs)))
        else:
            if kwonlyargs:
                specs.append("*")
        if kwonlyargs:
            for kwonlyarg in kwonlyargs:
                spec = formatargandannotation(kwonlyarg)
                if kwonlydefaults and kwonlyarg in kwonlydefaults:
                    spec += formatvalue(kwonlydefaults[kwonlyarg])
                specs.append(spec)
        if varkw is not None:
            specs.append(formatvarkw(formatargandannotation(varkw)))
        result = "(" + ", ".join(specs) + ")"
        if "return" in annotations:
            result += formatreturns(formatannotation(annotations["return"]))
        return result

# end funcutils.py


Alli is sharing her bed with her new sister Sophie.
Ms. Walker as she was known to you. Lovely dog.

Hello to all Staff at the Niagara Humane Society!
Just sending all you awesome people an update on Ursula's new home. I have renamed her SASHA.
She has adapted just awesome, and is such a cuddler! She is loving all the play room, toys and loving from myself and my mother.
I've attached a few photos of her in her new forever home for you to enjoy.
You all are such wonderful people and was impressed by the care you take with the animals. Sasha is the best! I can't imagine not having her in the family now. Such a wonderful fur baby! Keep up the awesome work!
Sincerely,
Michelle

Hi this is Todd ,My family adopted pebbles. We have decided to keep her name as pebbles. She has become a part of the family the moment we brought her home. She has adapted very well to her new environment and all our wonderful family and neighbors. She gets along with everyone as well as other dogs we know. She has been a pleasure to have around the house. We can't thank you enough for letting our family adopt pebbles.

Hi!
We adopted Tessa from the NFHS on the 20th of December and we can not be more happy with her!
She was not responding to Tessa and we were not overly fond of the name so we changed it to Freya which she is somewhat responding to now. :D
We have taken so many pictures of her and our other dog Cricket and they just play and play and play all day!
I hope it is not too early to send pictures.
I also hope it is okay to send pictures of Freya and her sister Cricket.
Freya is such a sweet dog, we are so happy to welcome her into our house and could not be more pleased with how well she is adapting to her surroundings and her sibling.
Thank you very much for doing what you do to help these dogs, without you we would never have such a wonderful dog.

Hey guys,
Just wanted to share an update on sloth and his adorable picture. He's doing well and is very happy, thanks again for nursing him back to health so he could be adopted.


---
abstract: 'We present a method to learn a joint multimodal representation space that enables recognition of unseen activities in videos. We first compare the effect of placing various constraints on the embedding space using paired text and video data. We also propose a method to improve the joint embedding space using an adversarial formulation, allowing it to benefit from *unpaired* text and video data. By using unpaired text data, we show the ability to learn a representation that better captures unseen activities. In addition to testing on publicly available datasets, we introduce a new, large-scale text/video dataset. We experimentally confirm that using paired and unpaired data to learn a shared embedding space benefits three difficult tasks (i) zero-shot activity classification, (ii) unsupervised activity discovery, and (iii) unseen activity captioning, outperforming the state-of-the-arts.'
author:
- |
    AJ Piergiovanni$^1$ Michael S. Ryoo$^{1,2}$\
    $^1$Indiana University, $^2$Stony Brook University\
    [{ajpiergi,mryoo}@indiana.edu]{}
bibliography:
- 'bib.bib'
title: Learning Multimodal Representations for Unseen Activities
---

Introduction
============

Videos contain multiple data sources, such as visual, audio and text/caption data. Each data modality has distinct statistical properties capturing different aspects of the event. Current state-of-the-art activity recognition models [@carreira2017quo; @tran2017closer] only take visual data and class labels as input, which limits the information the model can learn from. For example, the sentence ‘a group of men play basketball outdoors’ contains rich information, such as ‘outdoors’ and ‘group of men’ compared to just the activity class label of ‘basketball.’ We desire to use such additional information to learn better representations and by doing so, we show that the learned representations are able to generalize to unseen activities (i.e., zero-shot learning).

![Taking advantage of both text and video data allows for learning of a shared representation. By utilizing unpaired text and video data, the representation naturally captures the relationships between different activities, based on the underlying relationships in word embeddings and video representations. The colors represent different activity classes of the video or sentence (e.g., various sports, pool activities, and exercises).[]{data-label="fig:concept"}](figures/wacv-concept.pdf){width="\linewidth"}

We explore multimodal learning from video and language data, each starting with its own representation. Video data is represented as a sequence of images (spatio-temporal pixel data) while text is represented as a sequence of word embeddings (temporal data). Learning a shared representation allows for modeling the highly non-linear relationships between these modalities, capturing structure present in both video and textual data. Further, using a shared representation enables capturing similarities between concepts (e.g., basketball and volleyball both being sports with a ball) within its space by relying on either modality, even when the data is unpaired. This allows the representation to benefit from concepts not seen in both modalities during training. For example, we show taking advantage of relationships between words in pre-trained word embeddings [@mikolov2013efficient] help recognize activities with no video examples. By learning a shared representation space, we transfer such relationships to video representations of potentially unseen activities. An conceptual overview of the approach is shown in Fig. \[fig:concept\].

Many existing approaches to both zero-shot and embedding space learning require paired data examples (e.g., examples and labeled attributes), which can be expensive to obtain. By taking advantage of adversarial learning [@goodfellow2014generative], we are able effectively augment our method with **unpaired** data (i.e., random sentences and random videos without any labels or correspondence) to further improve our learned representation. By introducing many random videos and text data, we show that we are able learn representations that better capture unseen activities, without requiring any further annotations.

In this paper, we design a method capable of learning joint video/language representation using both paired and unpaired data. We experimentally confirm its benefit to three challenging tasks: (i) zero-shot activity recognition, (ii) unsupervised activity discovery, and (iii) unseen activity captioning. We show that the use of unpaired, multimodal data allows learning a shared embedding space that generalizes to unseen data.

![image](figures/nips-lang-model.pdf){width="95.00000%"}

Related works
=============

#### Multimodal learning

Previous approaches to multimodal learning have used Restricted Boltzmann Machines [@srivastava2012multimodal] or log-bilinear models [@kiros2014multimodal] to learn distributions over sentences and images. Ngiam et al. [@ngiam2011multimodal] designed an autoencoder that learns joint audio-video representations, however relied on greedy, layer-by-layer training instead of training the model end-to-end. Similarly, Chandar et al. [@chandar2016correlational] proposed an auto-encoder able to learn correlations between different view of images. Frome et al. [@frome2013devise] describe a model that maps images and words to a shared embedding. However, these works either learn a joint embedding by concatenating the different features or require a triplet consisting of positive and negative pairs; they have not explored the use/effect of unpaired data.

#### Text and vision

Using both text and visual data has been studied for many tasks, such as image captioning [@karpathy2014deep; @johnson2016densecap; @karpathy2015deep] or video captioning [@krishna2017dense; @zhou2018end; @xu2018joint]. Other works have explored the use of textual grounding for image/video retrieval [@gupta2010using; @rohrbach2016grounding; @miech2017learning; @anne2017localizing]. We note that using text for video retrieval/localization (e.g., [@anne2017localizing]) is similar in nature to the zero-shot or unseen recognition tasks. However, in those works, there is significant overlap between the text/video examples used in training and testing, while in our work we explicitly separate the classes used during training and evaluation; we focus on ‘unseen’.

There have been various models proposed to learn a fixed text embedding space with mappings from video features into this embedding space [@guadarrama2013youtube2text; @otani2016learning; @song2016unsupervised; @wang2016learning; @xian2016latent]. These works all learn a single directional mapping, without a shared representation space (which we find to be important). Further, most of them only learn with paired text/image samples and some require data in the form of positive/negative pairs. In this paper, we find learning a shared representation space and using unpaired, i.e., random additional data, to be important.

#### Learning with unpaired data

Recently, there have been many works taking advantage of variational autoencoders (VAEs) [@kingma2014auto] or generative adversarial networks (GANs) [@goodfellow2014generative] to learn mappings between unpaired samples. CycleGan [@zhu2017unpaired] uses a cycle-consistency loss (i.e., the ability to go from a sample in one domain to a second domain then back to the source) to learn unpaired image translation (e.g., image to sketch). Other works learn many-to-many mappings between images [@almahairi2018augmented] or use two GANs to map between domains [@yi2017dualgan]. An autoencoder with shared weights for both domains has been used to learn a latent space for image-to-image translation [@liu2017unsupervised]. However, these works all focus on learning mappings between unpaired data of the same modalitiy (e.g. image to image), where the data is from the same underlying distribution. We focus on a more challenging problem: learning from different modalities with very different distributions, where we find directly using previous approaches do not perform well as they are.

#### Zero-shot activity recognition

There are works on zero-shot activity recognition. Common approaches include using attributes [@liu2011recognizing; @palatucci2009zero; @romera2015embarrassingly] or word embeddings [@xu2015semantic; @xu2017transductive; @norouzi2013zero; @socher2013zero; @kodirov2017semantic] or learning a similarity metric [@zhang2015zero; @chopra2005learning]. Some works have explored using adversarial losses on the latent space [@chen2018zero], used GANs to generate features for unseen classes [@xian2018feature] or used auto-encoders [@wang2017zero]. Felix et al. [@felix2018multi] proposed a GAN-based approach to learn embeddings for zero-shot learning. Different from our approach, they applied the GAN only on the semantic, hand-crafted attributes of the classes to generate representations. We formulate a more general framework generating representations for all modalities, also taking advantage of more generic and challenging text and video.

Importantly, our work differs from these previous works in three key ways: (1) we show the benefit of using additional **unpaired** samples, (2) we experimentally compare the use of the representations for three tasks (i.e., zero-shot recognition, unseen recognition, and unseen video captioning), and (3) we learn a shared, multimodal representation with bi-directional mappings in an end-to-end fashion. We find that directly using the previous methods with unpaired data do not perform as well.

Method
======

To enable learning of a shared representation, we use a deep autoencoder architecture. Our model consists of 4 neural networks: $$\begin{aligned}
\mbox{\textbf{Video Encoder} } & E_{V}: v \mapsto z_v
&
\mbox{\textbf{Video Decoder} } & G_{V}: z \mapsto v
\\
\mbox{\textbf{Text Encoder} } & E_{T}: t \mapsto z_t
&
\mbox{\textbf{Text Decoder} } & G_{T}: z \mapsto t\end{aligned}$$ where $v$ is a sequence of video data and $t$ is a sentence (sequence of words). $z$ is the representation in the shared space that we are learning. The encoders learn a compressed representation of the video or text while the decoders are trained to reconstruct the input: $$\label{eq:recons}
    \mathcal{L}_{recons}(v,t) = ||G_V(E_V(v)) - v||_2 + ||G_T(E_T(t)) - t||_2$$ As both text and video data are sequences, they often have different lengths. A shared representation requires that the features from both modalities have the same dimensions. Given a text representation of length $L$ and a video representation of length $T$, we need to obtain a fixed-size representation. To learn a fixed-dimensional representation, there are many choices for the encoder/decoder architecture, such as temporal pooling [@ng2015beyond], attention [@piergiovanni2017learning] or RNNs [@krishna2017dense]. We chose temporal attention filters [@piergiovanni2017learning] as they learn a mapping from any length input to a $N$-dimensional vector and have been shown to outperform temporal pooling and RNNs on activity recognition tasks.

The attention filters consists of $N$ Gaussians, each learning 2 parameters: a center $\hat{g}$ and width $\sigma$, which are constrained to be positive. The filters are determined by: $$\label{eq:mrf_match}
\begin{split}
g_n &= 0.5\cdot T \cdot (\hat{g}_n + 1)\\
F[n,t] &= \frac{1}{Z} \exp(-\frac{(t-g_n)^2}{2\sigma_n^2})\\
&n\in\{0,1,\ldots,N-1\},~ t\in\{0,1,\ldots,T-1\} \\
\end{split}$$ The weights are applied by matrix multiplication with the video or text sequence (e.g., the outputs of $E_V$ or $E_T$): $v' = Fv$. This (i.e., $v'$) is then used as the representations for the joint space. Additionally, we can learn a transposed version of these filters to reconstruct the input: $v=F^Tv'$. To reconstruct the input, the decoders learn their own parameters with the tensors transposed, resulting in the matching output size. Fig. \[fig:model\] shows our encoder architecture.

Learning a joint embedding space
--------------------------------

![image](figures/nips-video-lang-paired.pdf){width="95.00000%"}

To learn a joint representation space, we minimize the $L_2$ distance between the embeddings of a pair of text and video (shown in Fig. \[fig:paired-data\](a)): $$\label{eq:joint}
    \mathcal{L}_{joint}(v,t) = ||E_V(v) - E_T(t)||_2$$ This forces the joint embeddings to be similar and when combined with the reconstruction loss, ensures that the representations can still reconstruct the input.

We can further constrain the networks and learned representation by forcing a cross-domain mapping from text to video and from video to text (shown in Fig. \[fig:paired-data\](b)): $$\label{eq:cross}
    \mathcal{L}_{cross}(v,t) = ||G_T(E_V(v)) - t||_2 + ||G_V(E_T(t)) - v||_2$$

Additionally, we can use a ‘cycle’ loss to map from video to text and back to video. Note that while the previous losses all require paired examples, this loss does not. $$\label{eq:cycle}
\begin{split}
    \mathcal{L}_{cycle}(v,t) &= ||G_T(E_V(G_V(E_T(t)))) - t||_2\\
    &+ ||G_V(E_T(G_T(E_V(v)))) - v||_2
\end{split}$$

To train the model to learn a joint embedding space, we minimize $$\label{eq:paired}
\begin{split}
\mathcal{L}(v,t) &= \mathcal{L}_{recons}(v,t) + \alpha_1\mathcal{L}_{joint}(v,t)  \\
&+ \alpha_2\mathcal{L}_{cross}(v,t) + \alpha_3\mathcal{L}_{cycle}(v,t)
\end{split}$$ where $\alpha_i$ are hyper-parameters weighting the various loss components.

Semi-supervised learning with unpaired data
-------------------------------------------

![Semi-supervised alignment with adversarial learning \[alg:train\]](figures/nips-video-lang-unpaired.pdf){width="\linewidth"}

Sample ($V$, $T$) from paired training data Update encoders/decoders (Eq. \[eq:paired\]) Update discriminators (Eq. \[eq:discrim\]) Sample $P=$($V_p$, $T_p$) from paired and $U=$($V_u$, $T_u$) from unpaired training data Update encoders/decoders with $P$ (Eq. \[eq:paired\]) Update encoders/decoders with $U$ (Eq. \[eq:gen\]) Update discriminators based on all (Eq. \[eq:discrim\])

To learn using unpaired data (i.e., unrelated text and video), we use an adversarial formulation. We treat the encoders and decoders as generator networks. We then learn an additional 3 discriminator networks which constrain the generators and embedding space and force the encoders and decoders to be consistent:

(1) $D_z$ which learns to discriminate between latent text representations and latent video representations. Conceptually, this constrains the learned embeddings to appear to be from the same distribution.

(2) $D_V$ which learns to discriminate between true video data and generated video data $G_V(E_T(t))$.

(3) $D_T$ which learns to discriminate between true text data and generated text data, $G_T(E_V(v))$.

Given these discriminators, we minimize the following losses: $$\label{eq:discrim}
\begin{split}
    \operatorname{\mathcal{L}}_{D_z}(v,t) &= - \log (D_z(E_T(t)))  - \log (1 -  D_z(E_V(v)))\\
    \operatorname{\mathcal{L}}_{D_V}(v,t) &= -\log (D_V(v)  - \log (1 -  D_V(G_V(E_T(t))))\\
    \operatorname{\mathcal{L}}_{D_T}(v,t) &= -\log (D_T(t))  - \log (1 -  D_T(G_T(E_V(v))))\\
\end{split}$$

Using the discriminators, we can train the generators (encoders and decoders) to minimize the following loss based on unpaired data: $$\label{eq:gen}
\begin{split}
    \operatorname{\mathcal{L}}_{G_z}(v,t) &= \log (D_z(E_T(t))))  + \log (1 -  D_z(E_V(v)))\\
    \operatorname{\mathcal{L}}_{G_V}(v,t) &= \log (1 -  D_V(G_V(E_T(t))))\\
    \operatorname{\mathcal{L}}_{G_T}(v,t) &= \log (1 -  D_T(G_T(E_V(v))))\\
\end{split}$$ Note that in this formulation, $v$ and $t$ are not paired.

These networks are trained in an adversarial setting. For example, for the text-to-video generator (i.e., $v' = G_V(E_T(t))$ and video discriminator, $D_V$, we optimize the following minimax equation: $$\begin{split}
    \min_{E_T,G_V} & \max_{D_V} = \operatorname{\mathbb{E}}_{v\sim p_{\text{data}}(v)} [\log D_V(v)] \\
    &+ \operatorname{\mathbb{E}}_{t\sim p_{\text{data}}(t)} [\log (1-D_V(G_V(E_T(t))))]
\end{split}$$

This equation is similarly applied for video-to-text. For learning the embedding space with the video and text encoders, $E_V,E_T$ and the discriminator $D_z$, we optimize the following minimax equation: $$\begin{split}
    \min_{E_T,E_V} & \max_{D_z} = \operatorname{\mathbb{E}}_{v\sim p_{\text{data}}(v)} [\log D_z(E_V(v))] \\
    &+ \operatorname{\mathbb{E}}_{t\sim p_{\text{data}}(t)} [\log (1-D_z(E_T(t)))]
\end{split}$$

As training GANs can be unstable, we developed a method to allow for more stable training of the joint embedding space, shown in Algorithm \[alg:train\]. We initialize both the generator and discriminator networks by training only on paired data. After several iterations of this, we train with both unpaired and paired data. We found the initial training of the generators and discriminators was important for stability, without it the loss often diverges and the learned embedding did not generalize to unseen activities.

[lcccc]{} & 5 Unseen & 10 Unseen & 20 Unseen & 50 Unseen\
Paired Data\
Fixed Text Representation & 41.9 & 38.4 & 29.4 & 15.6\
Triplet Loss & 56.8 & 44.9 & 38.8 & 23.3\
joint & 54.3 & 41.7 & 36.1 & 21.2\
recons + cross & 21.1 & 12.6 & 7.6 & 2.9\
joint + recons & 70.1 & 54.4 & 42.6 & 27.5\
joint + recons + cycle & 70.4 & 54.3 & 42.1 & 26.8\
joint + recons + cross & 72.6 & 55.4 & 43.2 & 27.8\
joint + recons + cross + cycle & 76.4 & 56.9 & 45.5 & 28.8\
triplet + recons + cross + cycle & 76.7 & 57.2 & 46.3 & 29.1\
\
+ $D_z$ & 78.5 & 57.4 & 45.9 & 29.3\
+ $D_v + D_t$ & 77.4 & 57.2 & 45.7 & 28.9\
+ $D_z + D_v + D_t$ & 79.8 & 58.4 & 46.5 & 29.8\
Paired + Unpaired Data\
recons + cycle & 22.8 & 13.6 & 8.4 & 4.2\
triplet + recons + cycle & 72.6 & 58.4 & 44.7 & 29.3\
triplet + recons + cross + cycle & 73.4 & 59.1 & 45.3 & 29.2\
Without Algorithm \[alg:train\] & 23.4 & 11.7 & 6.5 & 3.1\
All terms & 82.5 & 60.4 & 46.2 & 30.1\

Experiments
===========

We compare our various approaches on different tasks (i) zero-shot activity recognition, (ii) unsupervised activity discovery and (iii) unseen activity captioning. These tasks test various combinations of our encoders and decoders and how well the shared representation generalizes to unseen data. We experimentally confirm the benefits of our methods using multiple public datasets: AcitivtyNet [@heilbron2015activitynet; @krishna2017dense], HMDB [@kuehne2011hmdb], UCF101 [@soomro2012ucf101], and MLB-YouTube [@mlbyoutube2018]. Implementation details can be found in the Appendix.

#### Baselines

For baselines, we compare to a fixed-text embedding space, were only a mapping from video data into the text embedding space is learned (e.g., [@otani2016learning]). We also compare to learning a shared embedding space with the ‘recons’ (Eq. \[eq:recons\]) and ‘cross’ (Eq. \[eq:cross\]) terms (e.g., [@ngiam2011multimodal]). We additionally compare to methods like CycleGan [@zhu2017unpaired], using various components without Algorithm \[alg:train\].

Zero-shot activity recognition
------------------------------

Zero-shot activity recognition is the problem of classifying a video that belongs to a class not seen during training. Given training videos of seen classes together with paired text descriptions, our approach learns a shared embedding that maps videos/texts from multiple seen classes. The objective is to classify videos of unseen classes solely based on the learned embedding space and the text samples.

To enable recognition of unseen activities, we use a sentence of the new, unseen class and obtain its representation in the shared space. We can then obtain representations of videos in the same space, using nearest neighbors matching to classify each clip. Such approach takes advantage of the learned textual relationships (e.g., [@mikolov2013efficient]) and the shared, multimodal representation space.

We use the ActivityNet captions [@krishna2017dense] dataset to learn the shared representations, as this dataset has both sentence descriptions for each video as well as activity classes. We randomly choose a set of $K$ activity classes and withhold all videos/sentences belonging to those classes during training. For evaluating on the unseen activities, we take a subset of sentences for the unseen classes and map the sentences into the joint embedding space, $z_t = E_T(t)$. We then map the videos into the space, $z_v = E_V(v)$ and use nearest neighbors to match each video ($z_v$) to text ($z_t$), using the class of the nearest sentence as the classification for the video. We rely on the similarities between the representations (e.g., word embeddings) to enable the models ability to generalize to these unseen classes.

In Table \[res:actnet\], we compare the effect of the various loss components. For each method, we run 10 trials each with a different set of unseen activity classes and average the results. We find that previous methods of learning a fixed language embedding (e.g., [@socher2013zero; @xu2015semantic; @xu2017transductive]) are significantly outperformed by learning a joint representation. Previous methods learning embedding spaces without the ‘joint’ term (e.g. [@ngiam2011multimodal]), we found yield nearly random performance on these tasks, suggesting that forcing the representations to match in the embedding space is important. Further, adding the reconstruction, cross-domain, and cycle losses all improve performance. We also compare to a standard triplet loss (e.g., [@guadarrama2013youtube2text]) which requires positive/negative samples. We find that the triplet loss outperforms the ‘joint’ loss, but is surpassed by adding the ‘cycle’ and ‘cross’ terms, which use less data. We also compared using the triplet loss when combined with the other terms, finding a slight improvement over the joint term. Note using both the joint and triplet would be redundant, since the triplet loss contains the joint loss terms.

We also compare the various components of the adversarial loss. We compare to having just the adversarial loss on the representation ($D_z$), like [@chen2018zero], and compare just the adversary on the generated videos/sentences. We find the use of all terms is important for performance.

While previous works such as [@ngiam2011multimodal] can support learning with unpaired data, we find that the adversarial loss provides better results than just the ‘cycle’ and ‘recons’ terms, and further improves over training with just paired data. Further, we find that CycleGan-style approaches, without Algorithm \[alg:train\], fail on this task.

In Table \[res:previous\], we compare our approach to previous zero-shot learning methods on HMDB and UCF101. The paired training data for these models is drawn from ActivityNet with any classes belonging to HMDB or UCF101 withheld. The unpaired text data is sampled from Charades and the video data comes from either HMDB (when testing on UCF101) or UCF101 (when testing on HMDB). As HMDB and UCF101 have no text descriptions, we created a sentence description for each activity class (included in Appendix B). We find that the shared representation outperforms the previous approaches on these datasets and unpaired adversarial learning further improves performance.

                                Feat       HMDB51           UCF101
  ---------------------------- ------ ---------------- ---------------- -- --
  SJE [@akata2015evaluation]    IDT    $12.0\pm 2.6$    $9.3 \pm 1.7$
  ConSe [@norouzi2013zero]      IDT    $15.0 \pm 2.7$   $11.6\pm 2.1$
  ZSECOC [@qin2017zero]         IDT    $22.6\pm 1.2$    $15.1\pm 1.7$
  SE [@xu2015semantic]          IDT    $21.2\pm 3.0$    $18.6\pm 2.2$
  MRR [@xu2017transductive]     IDT    $24.1\pm 3.8$    $22.1\pm 2.5$
  SAE [@kodirov2017semantic]    I3D    $25.6\pm 3.2$    $25.4\pm 2.2$
  Ours (paired)                 IDT    $26.3 \pm 3.2$   $25.4 \pm 3.4$
  Ours (paired + unpaired)      IDT    $29.7 \pm 2.2$   $26.4 \pm 2.1$
  Ours (paired)                 I3D    $28.3 \pm 2.7$   $27.8 \pm 2.2$
  Ours (paired + unpaired)      I3D    $34.7 \pm 2.4$   $33.4 \pm 1.8$

  : Results on HMDB51 and UCF101 (accuracy) compared to previous state-of-the-art results. We find that learning a shared representation is beneficial and that augmented with unpaired data provides the best results.[]{data-label="res:previous"}

                                  Unseen   All
  ------------------------------ -------- ------
  Paired Data                      58.3    69.6
  + Random Wikipedia Sentences     55.8    66.4
  + Random Dictionary Defs.        56.3    68.2
  + Verb Dictionary Defs.          59.2    70.7
  + Random YouTube Videos          58.7    70.1
  + Verbs + Random Videos          60.3    71.2

  : Comparison of various source of unpaired data on ActivityNet with 10 unseen classes, values reported for both unseen classes and all (seen+unseen) classes. Results are accuracy, higher is better.[]{data-label="res:unpaired-sources"}

                                Accuracy   mAP
  ---------------------------- ---------- ------
  Baseline I3D features           23.4     32.6
  Fixed Text Representation       27.9     34.7
  joint                           34.5     41.6
  joint + recons                  37.9     43.7
  joint + recons + cycle          44.2     48.6
  joint + recons + cross          43.7     49.3
  triplet + recons + cross        43.9     49.5
  All (paired)                    48.4     51.2
  All (+ unrelated unpaired)      39.7     43.9
  All (+ related unpaired)        49.1     54.3

  : Comparison of unsupervised activity classification on MLB-YouTube.

\[res:mlb\]

[lcccc]{} & ActNet (10 unseen) & ActNet (50 unseen) & HMDB51 & UCF101\
Fixed Text Representation & 55.7 & 46.8 & 24.5 & 26.8\
Triplet Loss & 57.7 & 48.5 & 27.6 & 29.8\
joint & 62.1 & 50.2 & 29.8 & 30.6\
joint + recons & 64.4 & 52.6 & 30.4 & 31.3\
joint + recons + cross + cycle & 69.6 & 58.5 & 35.6 & 36.5\
triplet + recons + cross + cycle & 69.8 & 58.6 & 35.7 & 36.8\
\
All terms & 71.7 & 65.9 & 38.9 & 42.2\

Use of Unpaired Data
--------------------

We explore different strategies for obtaining unpaired data. Keeping a fixed set of paired text and videos, we explore adding various sources of unpaired data: (i) 10k random Wikipedia sentences, (ii) 10k random dictionary definitions, and (iii) 10k verb dictionary definitions. We also compare adding 10k random videos from YouTube as additional video data. Ours results using 10 unseen classes are in Table \[res:unpaired-sources\]. We find that augmenting with similar unpaired data improves performance, while irrelevant data harms performance. We find that dictionary verb definitions improve performance the most, as they capture important semantic information regarding the activities we are learning. The use of additional video data is further beneficial.

Unsupervised activity discovery
-------------------------------

To further evaluate the shared representation, we conducted experiments on unsupervised activity discovery. For this task, we expanded the MLB-YouTube dataset [@mlbyoutube2018] by densely annotating the videos with a transcription of the announcers’ commentary, resulting in approximately 50 hours of aligned text and video. Examples of this data are shown in Fig. \[fig:baseball-ex-caption\]. The MLB-YouTube dataset is designed for fine-grained activity recognition, where the difference between activities is quite small. Additionally, these captions only roughly describe what is happening in the video, and often contain unrelated stories or commentary on a previous event, making this a challenging task. The dataset will be made publicly available. To train the shared representation, we split each baseball video into 30 second intervals and use the corresponding text as paired data, resulting in 6,089 paired training samples.

![image](figures/baseball-caption-ex.pdf){width="\linewidth"}

![image](figures/unseen-captioning.pdf){width="\textwidth"}

We evaluate the shared representation using the segmented videos from MLB-YouTube. For each video, we compute the embedded features and apply $k$-means clustering ($k=8$, the number of classes). Each segmented video is assigned to a cluster and votes for the cluster label based on its ground truth label. We use that cluster assignment for classification on the MLB-YouTube test set. We report our findings in Table \[res:mlb\]. As a baseline, we cluster I3D features pre-trained on Kinetics. We find that our methods improve the representation. However, we note that when using unpaired data from Charades, the performance drops. This is likely due to Charades data being very different from MLB-YouTube data. We collected additional captions and baseball videos to augment the MLB-YouTube dataset, and confirmed that unpaired data helps when it is from a similar distribution.

In Table \[res:unsupervised\] we compare various methods for unsupervised activity discovery on HMDB and UCF101. Here, we learn a shared representation using the ActivityNet videos and captions. We withhold any videos belonging to a class in HMDB or UCF101. Unlike MLB-YouTube, on these datasets, we find that using the unpaired training with Charades further improves performance. This confirms that when the additional data is similar to the target dataset, using the adversarial learning setting further improves the representations.

                      HMDB   UCF101
  ------------------ ------ --------
  I3D features        26.6    42.5
  Joint               32.4    57.7
  Joint + recons      33.5    59.0
  All (paired)        34.6    59.5
  All (+ unpaired)    34.9    59.9

  : Comparison of unsupervised activity classification on HMDB and UCF101.

\[res:unsupervised\]

Unseen video captioning
-----------------------

As our model learns a bi-directional mappings, we can apply our model to generate video captions. Existing video captioning models are unable to create realistic captions for unseen activities, as without training data they do not know the words to describe the video. Given a video, $v$, we can generate a caption by mapping the video to text $t=G_T(E_V(v))$. For each word, we then use nearest neighbors matching with the GloVe embeddings to obtain the words to form a sentence. We find that using our method with paired and unpaired data improves performance using METEOR (3.6 to 6.9) [@banerjee2005meteor] and CIDEr [@vedantam2015cider] (8.9 to 13.9) scores. For these metrics, higher values are better and are measured with the unseen classes from the ActivityNet dataset. In Table \[res:caption\], we report the commonly used METEOR [@banerjee2005meteor] and CIDEr [@vedantam2015cider] scores of our various models, measured with the unseen classes from the ActivityNet dataset. We find that learning a shared representation (4.1) is beneficial and using unpaired samples further improves the task (5.3 paired only vs 6.9 paired and unpaired). In Fig. \[fig:unseen-caps\], we show example captioned videos. Note that this task is extremely challenging, as it requires the model to generate captions using activity words (e.g., basketball) not seen during training.

                               METEOR   CIDEr
  --------------------------- -------- -------
  Fixed Text Representation     3.64    8.95
  Joint                         4.21    9.23
  All (paired)                  5.31    11.21
  All (paired + unpaired)       6.89    13.95

  : Comparison of several models for unseen activity captioning using the ActivityNet dataset, using METEOR and CIDEr scores. This evaluation was done on 10 unseen classes held out during training. Higher values are better.[]{data-label="res:caption"}

Conclusion
==========

We proposed an approach to learn a joint language/text representation using various constraints. We further extended the model to be able to learn with unpaired video and text data using an adversarial formulation. We experimentally confirmed that learning with unpaired data is beneficial to three difficult tasks (i) zero-shot activity classification, (ii) unsupervised activity discovery, and (iii) unseen activity captioning. We find that the use of related unpaired data is beneficial. We presented several strategies for obtaining unpaired data and confirmed the benefit of adding additional, relevant unpaired data.

Implementation/training details
===============================

We implement our models in PyTorch. For the per-segment video CNN, we use I3D [@carreira2017quo] to obtain a $1024\times T$ video representation. We trained a version of I3D based on Kinetics-600, but withheld all classes that appear in ActivityNet, HMDB51, or UCF101 so that the classes are truly unseen. This resulted in a training set with 478 classes and 278k videos. Since generating videos is an extremely challenging task, the video autoencoders start with and generate the I3D feature. We use GloVe word embeddings [@pennington2014glove] to obtain a language representation. We set $N=4$ for the temporal attention filters and apply 4 fully connected layers. These layers are followed by $L_2$ normalization so that the embedding space has unit length [@tygert2015convolutional]. We train the models for 200 epochs and use stochastic gradient descent with momentum to minimize the loss function with a learning rate of 0.01. After every 50 epochs, we decay the learning rate by a factor of 10. When training in the adversarial setting (e.g., Algorithm 1 in the main paper), we initialize the network training for 50 epochs on paired data followed by 200 on the paired + unpaired data.

Unseen video captioning
-----------------------

As our model learns a bi-directional mappings, we can apply our model to generate video captions. Existing video captioning models are unable to create realistic captions for unseen activities, as without training data they do not know the words to describe the video. Given a video, $v$, we can generate a caption by mapping the video to text $t=G_T(E_V(v))$. For each word, we then use nearest neighbors matching with the GloVe embeddings to obtain the words to form a sentence. In Table \[res:caption\], we report the commonly used METEOR [@banerjee2005meteor] and CIDEr [@vedantam2015cider] scores of our various models, measured with the unseen classes from the ActivityNet dataset. We find that learning a joint representation is beneficial and using unpaired samples further improves the task. Note that this task is extremely challenging, as it requires the model to generate captions using activity words (e.g., basketball) not seen during training.

                               METEOR   CIDEr
  --------------------------- -------- -------
  Fixed Text Representation     3.64    8.95
  Joint                         4.21    9.23
  All (paired)                  5.31    11.21
  All (paired + unpaired)       6.89    13.95

  : Comparison of several models for unseen activity captioning using the ActivityNet dataset, using METEOR and CIDEr scores. This evaluation was done on 10 unseen classes held out during training. Higher values are better.[]{data-label="res:caption"}

Additional Experiments
======================

Comparison of temporal pooling methods
--------------------------------------

To confirm that temporal attention is beneficial, we compare different forms of temporal pooling (i) max-pooling, (ii) sum-pooling, (iii) LSTM, and (iv) temporal attention filters [@piergiovanni2017learning]. In Table \[tab:temporal\], we compare these temporal pooling methods learning the joint embedding space. We confirm that using the temporal attention filters performs best.

                                Accuracy
  ---------------------------- ----------
  Max Pooling                     23.4
  Sum Pooling                     24.1
  LSTM                            42.3
  Temporal Attention Filters      55.2

  : Comparison of temporal pooling methods for 5 unseen classes in the ActivityNet dataset.[]{data-label="tab:temporal"}

Comparison of different ratios of paired and unpaired data
----------------------------------------------------------

We compare different ratios of paired and unpaired data to see how much paired data we require and how much unpaired data is beneficial. For these experiments, we use all the loss terms (i.e., what provided us the best results). Note that in these experiments, the total number of samples was the same for each method (40k examples) so that we can directly compare the effects of unpaired data vs. paired data. Thus not all the available data was used.

In Table \[tab:ratio\], we show the results. We find that using no paired data results in nearly random performance, but using using some paired data greatly improves the embedding space. The model using 100% paired data performs best, as all the others are using less overall paired data.

  Paired/Unpaired    Accuracy
  ----------------- ----------
  100% / 0%            74.2
  75% / 25%            73.2
  50% / 50%            69.7
  25% / 75%            62.6
  0% / 100%            24.5

  : Comparison of different ratios of paired and unpaired data methods for 5 unseen classes in the ActivityNet dataset.[]{data-label="tab:ratio"}

We also compare augmenting our 40k paired training samples with different amounts of unpaired data. Since UCF101 and HMDB only have 13k and 7k examples, to get up to 60k samples, we also use videos from the Kinetics dataset [@carreira2017quo]. The results, shown in Table \[tab:ratio2\], show that adding the initial 10k samples is most beneficial, while additional samples do not seem to meaningfully improve results. However, due to our training method where each batch consists of 50% paired data and 50% unpaired data, the additional unpaired data does not harm results either.

  Unpaired Samples    Accuracy
  ------------------ ----------
  0                     77.1
  10k                   82.4
  20k                   83.9
  40k                   83.6
  60k                   83.5

  : Comparison using 40k paired examples and varying amounts of unpaired samples for 5 unseen classes in the ActivityNet dataset.[]{data-label="tab:ratio2"}

MLB-Youtube Captions
--------------------

![image](figures/nips-baseball-clustering.pdf){width="90.00000%"}

In Fig. \[fig:baseball-embed\], we compare t-SNE embeddings of the fixed text representation and our joint embedding space. This visually shows that learning a joint embedding space gives more distinct class distributions.

### MLB-YouTube Captions

As a baseline for the MLB-YouTube captions dataset, we compared several different models for standard video captioning (i.e., all activity classes are seen). This task is quite challenging compared to other datasets as the announcers commentary is not always a direct description of the current events. Often the announcers tell loosely related stories and attempt to describe events differently each time to avoid repetition. Additionally, the descriptions contain on average 150 words for each 30 second interval and current captioning approaches usually only trained and tested on 10-20 word sentences. Due to these factors, this task is quite challenging the standard evaluation metrics do not account for these factors. In Table \[res:caption-mlb\], we report our results on this task.

                               Bleu   METEOR   CIDEr
  --------------------------- ------ -------- -------
  Fixed Text Representation    0.12    0.04    0.12
  Joint Representation         0.14    0.08    0.15
  Joint + all paired           0.15    0.10    0.18
  Joint + paired + unpaired    0.10    0.02    0.08

  : Comparison of several models for standard, seen video captioning using the MLB-YouTube dataset, using Bleu, METEOR and CIDEr scores. Higher values are better.[]{data-label="res:caption-mlb"}

HMDB and UCF101 Sentences {#app:sent}
=========================

For the HMBD and UCF101 datasets, we created sentences to describe each activity class. Our sentences descriptions are included in this appendix.

These sentences are written for each activity class (by randomly selecting a single video per class) and are shared for all instances of the activity. Depending on what video was randomly chosen for the class, some sentences describe the actor as a ‘man’, ‘woman’, or ‘person’ which could confuse the model. Ideally, the CNN embedding needs to learn to ignore the impact of such pronoun changes.

We conducted experiments comparing randomly replacing the pronouns to determine if there was any bias introduced by the pronouns. We show the results in Table \[tab:pronouns\]. We find that the choice of pronouns does not impact performance, as our model automatically learns to focus more on verbs rather than pronouns. When examining the temporal attention filters on the sentences, we found that they placed very little ‘attention’ on the start of the sentence, where the pronoun usually is, suggesting that the pronoun has very little effect on the embedding space we learned.

                        Accuracy
  -------------------- ----------
  Baseline Sentences      33.4
  All ‘man’               33.2
  All ‘woman’             33.3
  All ‘person’            33.4
  Random pronoun          33.4

  : Comparison of various pronouns on the UCF101 dataset with 50 unseen classes.[]{data-label="tab:pronouns"}

HMDB:

1.  chew: a woman is chewing on bread

2.  golf: a man swings a golf club

3.  sword exercise: a person is playing with a sword

4.  walk: a person is walking

5.  jump: a person jumps into the water

6.  pour: a man pours from a bottle

7.  laugh: a man is laughing

8.  shoot gun: a person rapidly fires a gun

9.  run: a person is running

10. turn: a person turns around

11. ride bike: a man is riding a bike on the street

12. swing baseball: a boy hits a baseball

13. draw sword: a person draws a sword

14. sit: a person sits in a char

15. fencing: two men are fencing

16. dribble: a boy dribbles a basketball

17. stand: a person stands up

18. pushup: a man does pushups

19. sword: two people are fighting with swords

20. pullup: a boy does pullups in a doorway

21. smile: a man smiles

22. shake hands: two people shake hands

23. shoot ball: a person shoots a basketball

24. kick: a person kicks another person

25. somersault: a person does a somersault

26. flic flac: a boy does a backflip

27. hug: two people hug

28. hit: a boy swings a baseball bat

29. dive: a person jumps into a lake

30. drink: a man drinks from a bottle

31. punch: a woman punches a man

32. wave: a person waves their hand

33. talk: a person is talking

34. kiss: a man and woman kiss

35. catch: a boy catches a ball

36. smoking: a woman smokes a cigarette

37. eat: a man eats pizza

38. throw: a person throws a ball

39. climb stairs: a man is running down the stairs

40. kick ball: a person kicks a soccer ball

41. ride horse: a girl is riding a horse

42. fall floor: a man is pushed onto the ground

43. brush hair: a girl is brushing her hair

44. situp: a man does situps

45. cartwheel: a guy runs and jumps and flips

46. pick: a man picks a book

47. push: a boy pushes a table

48. climb: a man is climbing up a wall

49. handstand: three girls do handstands

50. clap: a woman claps her hands

51. shoot bow: a person shows a bow and arrow

UCF101:

1.  MilitaryParade: people are marching and waving a flag

2.  TrampolineJumping: kids are jumping on a trampoline

3.  PlayingDaf: a person moves a circle and hits it

4.  SalsaSpin: poeple are dancing and spinning

5.  CuttingInKitchen: a person is in the kitchen using a knife

6.  ApplyEyeMakeup: a woman is putting on makeup

7.  PlayingViolin: a person plays the violin

8.  YoYo: a person plays with a yoyo

9.  PlayingCello: a person is playing the cello

10. Bowling: a person is bowling

11. UnevenBars: a woman is spinning and flying on bars

12. BalanceBeam: a woman is on the balance beam

13. SkyDiving: people are falling out of the sky

14. SumoWrestling: two fat people are wrestling

15. PushUps: a man does pushups

16. FloorGymnastics: a girl does gymnastics

17. ApplyLipstick: a woman is putting on lipstick

18. BreastStroke: a woman is swimming

19. GolfSwing: a man swings a golf club

20. PlayingDohl: a person hits on a drum

21. HorseRiding: a woman rides a horse

22. PlayingFlute: a person blow into a flute

23. PizzaTossing: a man is making a pizza

24. CleanAndJerk: a person is lifting weights

25. WritingOnBoard: a person is writing on the wall

26. CricketShot: a person hits a ball with a bat

27. FieldHockeyPenalty: a girl in the field shoots a ball

28. HammerThrow: a person spins and throws an object

29. BodyWeightSquats: a man is squatting

30. CliffDiving: a person jumps off a cliff

31. Typing: a person is typing at a computer

32. MoppingFloor: a man mops the floor

33. TaiChi: people are doing tai chi

34. PlayingPiano: a person plays piano

35. Punch: someone punches another person

36. Nunchucks: a person swings nun chucks

37. RopeClimbing: a person climbs a rope

38. Swing: a baby is swinging

39. Knitting: a woman is knitting

40. Rafting: people are rafting on a river

41. PlayingGuitar: a person strums a guitar

42. ShavingBeard: a man shaves his beard

43. JugglingBalls: a person is juggling balls

44. Diving: a boy dives into a pool

45. JumpingJack: a person jumps and swings his arms

46. VolleyBallSpiking: people hit a volleyball

47. PoleValut: a person runs with a pole and launches into the air

48. SkateBoarding: a man is skateboarding

49. BoxingPunchingBag: a man is punching a bag

50. IceDancing: people are ice skating

51. WallPushups: a person does pushups against a wall

52. FrisbeeCatch: a person jumps and catches a frisbee

53. Drumming: people are drumming

54. JumpRope: a girl is jumping rope

55. HeadMassage: a person gets their head massaged

56. PlayingTabla: a person plays two drums

57. TableTennisShot: people are playing table tennis

58. PommelHorse: a person spins around on their hands

59. HighJump: a man jumps over a bar and lands on his back

60. BasketballDunk: a man jumps and dunks the basketball

61. BoxingSpeedBag: a man punches a bad in the air quickly

62. PullUps: a person does hangs on a bar and pulls up

63. RockClimbingIndoor: a person is climbing up rocks

64. BlowingCandles: a boy blows out candles on a cake

65. Skiing: people are skiing on a mountain

66. WalkingWithDog: a person walks a dog

67. Basketball: men are playing basketball

68. SoccerJuggling: a person is playing with a soccer ball

69. Fencing: people are fencing

70. Billiards: a man is playing billiards

71. BaseballPitch: a man throws a baseball

72. BlowDryHair: a woman is drying her hair

73. CricketBowling: a person throws a cricket ball

74. BandMarching: people are walking down the street playing music

75. PlayingSitar: a person plays a funny guitar

76. ThrowDiscus: a person spins and throws a disk

77. StillRings: a man holds in the air on rings

78. Lunges: a person bends to the ground with one knee

79. Skijet: a person rides a jetski in the ocean

80. BabyCrawling: a baby is crawling on the floor

81. Mixing: a woman is mixing in a bowl

82. Hammering: a person is hitting nails with a hammer

83. Shotput: a person spins and launches a ball

84. Archery: a man shoots a bow and arrow

85. Surfing: a man is surfing in the ocean

86. FrontCrawl: a person is swimming freestyle

87. HulaHoop: a person spins a hoop around their waist

88. JavelinThrow: a person throws a spear

89. Rowing: people are in a canoe and rowing

90. Kayaking: a person is kayaking on a lake

91. ParallelBars: a man does gymnastics on the parallel bars

92. HorseRace: horses are racing around a track

93. HandstandWalking: a person stands on their hands and walk

94. BrushingTeeth: a boy brushes his teeth

95. LongJump: a person runs and jumps into a sand pit

96. Biking: people are riding bikes

97. HandstandPushups: a person does pushups upside down

98. BenchPress: a man is lifting weights

99. Haircut: a person is getting a hair cut

100. TennisSwing: a woman hits a tennis ball