\begin{table}[]
    \centering
    \resizebox{\columnwidth}{!}{
    \begin{tabular}{lcccccc}\hline
     \textbf{Dissimilarity} & \textbf{Diverse} & \textbf{Taxonomy} & \textbf{Questions}  \\\hline
        Answers  & \underline{0.567} (0.009) & \underline{0.791} (0.000) & \underline{0.795} (0.000) \\
        Image embed. &  0.248 (0.293) & \underline{0.492} (0.028) & \underline{-0.640} (0.002)) \\
        Question embed. & 0.184 (0.437) & \underline{0.531} (0.016) & \underline{0.631} (0.003) \\
        Joint embed. & 0.220 (0.350) &   \underline{0.622} (0.003) & -0.223 (0.344) \\\hline
    \end{tabular}}
    \caption{Spearman correlation of pairwise performance drop and and different dissimilarity heuristics. In addition to the results in Table~\ref{tab:spearman}, we show in parentheses the corresponding p-values. We underline statistically significant results (p < 0.05).}
    \label{tab:spearman_appendix}
\end{table}
\section{Introduction}\label{sec:intro}
The current paradigm to approach Vision+Language (V+L) tasks is to pretrain large-scale models, which are then finetuned and evaluated on independent and identically distributed (i.i.d.) data.
In practice, the i.i.d. assumption does not hold:
New data becomes available over time, which often results in a shift in data distribution.
One solution is to continuously adapt an existing model via finetuning. However, this will lead to catastrophic forgetting, i.e.\ significant performance degradation on previous data~\cite{mccloskey1989catastrophic,ratcliff1990connectionist}.
Continual learning provides a counterpart to i.i.d. learning: It defines a class of algorithms aiming at incremental learning with minimal forgetting.
This line of work becomes increasingly relevant given the financial and environmental costs of (re-)training large models~\cite{strubell-etal-2019-energy,bender2021dangers}, and the limited generalization of static models~\cite{lazaridou2021pitfalls}.

\subsubsection{Language Setting}\label{sec:q_splits}
We create a third setting {\em Question Types}, where each task corresponds to learning to answer a different category of questions.
We use a classification scheme developed by~\citet{whitehead2021separating}
to form a sequence of five tasks: 
Count, Color, Scene-level, Subcategory, and Action recognition. The splits for Count, Color, and Subcategory questions are obtained from~\citet{whitehead2021separating}. We create two additional tasks from the remaining questions.
In particular, we cluster question embeddings from Sentence-BERT~\cite{reimers-2019-sentence-bert} \footnote{We use the `all-MiniLM-L6-v2' model and Fast Clustering algorithm from the sentence-transformers package (\url{https://www.sbert.net/}).} so that each cluster has at least 15 questions and a minimum cosine similarity of 0.8 between all embeddings.
We annotate clusters as `scene', `action' or `irrelevant' question types.
Based on a seed of 10K annotated questions, we retrieve all other questions with similarity above 0.8 and label them using the K-nearest neighbor algorithm ($K=5$).  
Question Types have a total of 140K train, 35K validation and 84K test samples (cf.\ Table~\ref{tab:stats}). 
Common question words and answers per task are presented in the Appendix (Figure~\ref{fig:top_ans_q}). 

\subsection{Continual Learning Methods}
We benchmark common continual learning algorithms, including regularization- and replay-based approaches. 
We investigate two regularization-based approaches:
{\em Learning without Forgetting} (LwF)~\cite{li2017lwf}, which uses knowledge distillation~\cite{hinton2015distilling} in order to retain knowledge from previous tasks, and  
{\em Elastic Weight Consolidation} (EWC)~\cite{kirkpatrick2017elastic}. The EWC regularization term discourages big changes of parameters that were important for previous tasks, where importance is approximated using the Fisher information matrix.

We also experiment with a baseline Pseudo-Replay method for the Question Types setting.
Instead of storing raw data from previous tasks, we use a data augmentation method, inspired by~\cite{kafle2017data,kil2021discovering}. 
When training on task $t$, we augment the data $D_t$ by retrieving past questions based on their shared detected objects classes. For example, if an elephant is detected on the current picture, we retrieve a past question about an elephant. We then use the previous model $f_{\theta_{t-1}}$ to generate a distribution $\tilde{ \bm{y}} = f_{\theta_{t-1}}(\tilde{\bm{x}})$ which serves as soft targets for the new sample $\tilde{\bm{x}}$.
By not storing the original answers, we address privacy and efficiency concerns of replay approaches~\cite{van2018generative,Delangesurvey}.

\begin{equation}
   S_{T,i} = \frac{1}{N} \sum_{j=1}^{N} (1-\mathrm{cos}(\bm{e_{Tj}}, \bm{e_{ij}}))\cdot \Delta^{Ti}_j
\end{equation}

Next, we compute the Spearman correlation between the relative accuracy drops and different factors of task dissimilarity.
Here, we consider the answer distributions , as well as average embeddings of the image, question and the joint pair.
Consider $P$, $Q$  the answer distributions of Tasks $T_1,T_2$ respectively.
Since some answers of $T_1$ do not appear in $T_2$, we measure the skew divergence~\cite{pmlr-vR3-lee01a} between $P$ and $Q$ as the KL divergence between $P$ and a mixture distribution \mbox{$(1-\alpha) P + \alpha Q$} with $\alpha=0.99$~\cite{ruder-plank-2017-learning}.
For the input embeddings, we measure the cosine distance between the average task representation. 
As image representations, we utilize Faster R-CNN features from~\cite{Anderson2017up-down}, while questions are embedded using Sentence-BERT.
Joint embeddings for image-question pairs are obtained using the final layer representation of the [CLS] token of UNITER~\footnote{[CLS] is the first token of the input sequence which aggregates multimodal information. Its representation from the final encoder layer is passed to the classifier to predict an answer.}.
The detailed similarity measures are shown in the Appendix Figure~\ref{fig:dissimilarity}.

\section{Implementation Details} \label{sec:implementation_details}
Our implementation is based on the publicly available PyTorch codebase of UNITER (\url{https://github.com/ChenRocks/UNITER}).
For the continual learning experiments, we train a UNITER-base model (86M parameters) on a cluster of NVIDIA V100 GPUs using a single node with 4 GPUs.
Training on a sequence of 5 tasks requires on average $\sim$ 5 GPU hours. The main experiments (Table~\ref{tab:results}) require approximately a total of 200 GPU hours.

\begin{itemize}[leftmargin=*]
 \setlength\itemsep{1pt}
    \item \textbf{Diverse Domains}
    \item group 5, group 3, group 2, group 4, group 1
    \item group 1, group 2, group 5, group 3, group 4
    \item group 4, group 3, group 5, group 1, group 2
    \item group 3, group 1, group 4, group 2, group 5
    \item group 2, group 5, group 1, group 4, group 3
    \item \textbf{Taxonomy Domains}
    \item food, animals, sports, interior, transport
    \item transport, sports, food, animals, interior
    \item interior, animals, food, transport, sports
    \item animals, food, interior, sports, transport
    \item sports, interior, transport, animals, food
    \item \textbf{Question types}
    \item action, count, subcategory, scene, color
    \item color, subcategory, action, count, scene
    \item scene, count, action, color, subcategory
    \item subcategory, color, scene, action, count
    \item count, scene, color, subcategory, action
\end{itemize}
{"value": "0.567", "type": "Other"} {"value": "0.791", "type": "Other"} {"value": "0.795", "type": "Other"} {"value": "0.248", "type": "Other"} {"value": "0.492", "type": "Other"} {"value": "-0.640", "type": "Other"} {"value": "0.184", "type": "Other"} {"value": "0.531", "type": "Other"} {"value": "0.631", "type": "Other"} {"value": "0.220", "type": "Other"} {"value": "0.622", "type": "Other"} {"value": "-0.223", "type": "Other"}