\begin{table}
    \centering
    \resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}\hline
    \multicolumn{4}{c}{{\bf w/o Pretraining}}  \\\hline
        \textbf{Method}  & \textbf{What animal} & \textbf{What room} & \textbf{What sport} \\\hline
        Finetuning &  33.09 \small{\textcolor{black}{$\pm$ 13.38}} & 54.38 \small{\textcolor{black}{$\pm$ 32.42}}  &  25.14 \small{\textcolor{black}{$\pm$ 32.11}} \\
        EWC & 48.18 \small{\textcolor{black}{$\pm$ 15.67}} & 83.48 \small{\textcolor{black}{$\pm$ 7.61}}  & 62.81 \small{\textcolor{black}{$\pm$ 13.67}} \\
        ER & 73.11 \small{\textcolor{black}{$\pm$ 0.70}} & 89.04 \small{\textcolor{black}{$\pm$ 2.80}}  & 87.20 \small{\textcolor{black}{$\pm$ 1.84}} \\\hline
    \multicolumn{4}{c}{{\bf w/ Pretraining}}  \\\hline
        \textbf{Method}  & \textbf{What animal} & \textbf{What room} & \textbf{What sport} \\\hline
        Finetuning &   75.07 \small{\textcolor{black}{$\pm$ 3.54}} & 83.26 \small{\textcolor{black}{$\pm$ 12.47}}  &  69.92 \small{\textcolor{black}{$\pm$ 14.14}}\\
        EWC & 81.75 \small{\textcolor{black}{$\pm$ 1.42}} & 94.32 \small{\textcolor{black}{$\pm$ 0.88}}  & 90.82 \small{\textcolor{black}{$\pm$ 1.36}}  \\
        ER & 80.73 \small{\textcolor{black}{$\pm$ 0.37}} & 94.10 \small{\textcolor{black}{$\pm$ 1.39}}  &  90.92 \small{\textcolor{black}{$\pm$ 0.71}}  \\\hline
    \end{tabular}}
    \caption{Accuracy and standard deviation of the best performing models on different sub-questions in Taxonomy Domains.}
    \label{tab:tax_task_order}
\end{table}
\section{Settings for Continual VQA} \label{sec:settings}
\subsection{Problem formulation}
In continual learning, model parameters $\bm{\theta}$ are incrementally updated as new data become available.
We assume that samples from tasks $t=1\dots T$ arrive sequentially as $D_t=\{\bm{x}_i, \bm{y}_i\}_{i=1}^{N_t}$, where $N_t$ is the number of data for task $t$.
Following previous work, VQA is formulated as a multi-label classification problem with soft targets $\bm{y}_i$~\cite{Anderson2017up-down}.
Starting from parameters $\bm{\theta}_{t-1}$ of the previous model, the updated parameters $\bm{\theta}_{t}$ are obtained by training on the new data $D_t$. 
Some approaches also use a memory $M_t$ containing a subset of samples from previous tasks, e.g. $D_{1}, \dots, D_{t-1}$.
In our setup, all tasks share a common output head which is extended with new classes from each task.
This allows inference to be task-agnostic but creates a more challenging setting than multi-head learning where separate heads are learned for each task~\cite{hussain2021towards}.
At the end of the training sequence, the objective is to achieve strong performance across all tasks observed so far. 
This objective encloses two challenges: 1) minimizing catastrophic forgetting of tasks seen earlier in training, 2) facilitating positive transfer to improve performance on new tasks~\cite{HADSELL20201028embracing}. 

\subsubsection{Language Setting}\label{sec:q_splits}
We create a third setting {\em Question Types}, where each task corresponds to learning to answer a different category of questions.
We use a classification scheme developed by~\citet{whitehead2021separating}
to form a sequence of five tasks: 
Count, Color, Scene-level, Subcategory, and Action recognition. The splits for Count, Color, and Subcategory questions are obtained from~\citet{whitehead2021separating}. We create two additional tasks from the remaining questions.
In particular, we cluster question embeddings from Sentence-BERT~\cite{reimers-2019-sentence-bert} \footnote{We use the `all-MiniLM-L6-v2' model and Fast Clustering algorithm from the sentence-transformers package (\url{https://www.sbert.net/}).} so that each cluster has at least 15 questions and a minimum cosine similarity of 0.8 between all embeddings.
We annotate clusters as `scene', `action' or `irrelevant' question types.
Based on a seed of 10K annotated questions, we retrieve all other questions with similarity above 0.8 and label them using the K-nearest neighbor algorithm ($K=5$).  
Question Types have a total of 140K train, 35K validation and 84K test samples (cf.\ Table~\ref{tab:stats}). 
Common question words and answers per task are presented in the Appendix (Figure~\ref{fig:top_ans_q}). 

We also experiment with a baseline Pseudo-Replay method for the Question Types setting.
Instead of storing raw data from previous tasks, we use a data augmentation method, inspired by~\cite{kafle2017data,kil2021discovering}. 
When training on task $t$, we augment the data $D_t$ by retrieving past questions based on their shared detected objects classes. For example, if an elephant is detected on the current picture, we retrieve a past question about an elephant. We then use the previous model $f_{\theta_{t-1}}$ to generate a distribution $\tilde{ \bm{y}} = f_{\theta_{t-1}}(\tilde{\bm{x}})$ which serves as soft targets for the new sample $\tilde{\bm{x}}$.
By not storing the original answers, we address privacy and efficiency concerns of replay approaches~\cite{van2018generative,Delangesurvey}.

\subsection{Evaluation Metrics}
After training on task $t$, we compute the VQA accuracy $A_{t,i}$ on data from the previous task $i$. 
We report the macro-average accuracy at the end of the training sequence: $\mathrm{A} = \frac{1}{T} \sum_{i=1}^T A_{T,i}$.
Following~\citet{riemer2018learning}, we report the learned accuracy $\mathrm{LA} = \frac{1}{T} \sum_{i=1}^T A_{i,i}$, which measures the ability to learn the new task $i$.
We also compute backward transfer $\mathrm{BWT} = \frac{1}{T-1} \sum_{i=1}^{T-1} A_{T,i} - A_{i,i}$~\cite{lopez2017GEM}, that captures the impact of catastrophic forgetting.

\begin{equation}
   S_{T,i} = \frac{1}{N} \sum_{j=1}^{N} (1-\mathrm{cos}(\bm{e_{Tj}}, \bm{e_{ij}}))\cdot \Delta^{Ti}_j
\end{equation}

\section{Results}
\subsection{Main Results}
\paragraph{Task Settings.}
Table~\ref{tab:results} summarizes the results averaged over five task orders using the UNITER backbone.
The results show an increasing difficulty for the three incremental learning task definitions, i.e. Diversity Domains $<$ Taxonomy Domains $<$ Question Types, which we will further investigate in Section \ref{sec:task_relationships}. 

Previous work on task-incremental learning for image classification~\cite{Yoon2020Scalable} has discussed the impact of task order to final performance, especially when tasks are dissimilar.
Similarly, we observe a high standard deviation in the Question Types results of  Table~\ref{tab:results}. 
In order to investiagte this further, we plot the final accuracy of a pretrained model for five training sequences in Figure~\ref{fig:qtask_order}, each ending with a different task. 
Our results show that task order can lead to Finetuning accuracy that varies more than 15$\%$. Although EWC improves the average accuracy, there is still a 10$\%$ fluctuation depending on the order. However, replay-based methods are able to improve performance and mitigate the sensitivity to task order.

While Table~\ref{tab:results} shows low variance in Taxonomy Domains, we find high variance when examining the performance on specific questions. In particular, we find that certain question types, such as Animals, Interior, and Sports, have high variance. Table~\ref{tab:tax_task_order} reveals a standard deviation which is up to 30 times higher compared to the average results in Table~\ref{tab:results}.
High standard deviation across randomized task orders is problematic since models can have different behavior in practice despite similar (aggregated) performance.
In other words, the current task performance will highly depend on the previous task order, even though the overall  accuracy from the randomized trials appears similar.
{"value": "33.09", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "54.38", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "25.14", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "48.18", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "83.48", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "62.81", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "73.11", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "89.04", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "87.20", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/o Pretraining": "true"}} {"value": "75.07", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "83.26", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "69.92", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "Finetuning"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "81.75", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "94.32", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "90.82", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "EWC"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "80.73", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What animal", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "94.10", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What room", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}} {"value": "90.92", "type": "Result", "training data/set": "VQA-v2", "test data/set": "VQA-v2 What sport", "task": ["VQA", "visual question answering"], "metric": "Accuracy", "experimental settings": {"Method": "ER"}, "model": "UNITER", "model settings": {"w/ Pretraining": "true"}}