\begin{table}[h]
    \centering
\resizebox{\columnwidth}{!}{
\begin{tabular}{l|ccccc}
\multicolumn{6}{l}{{\bf Diverse Domains}} \\\hline
\diagbox[width=6em]{Task 1}{Task 2}
& Group 1 & Group 2 & Group 3 & Group 4 & Group 5 \\\hline
Group 1 & \cellcolor{gray!60} - & \cellcolor{red!5.47985781990521}  -6.58 & \cellcolor{red!4.344391785150075}  -5.21 & \cellcolor{SoftGreen!3.139810426540407}  -4.84 & \cellcolor{red!5.91182859399684}  -7.09 \\
Group 2 & \cellcolor{SoftGreen!9.010600706713689}  -4.55 & \cellcolor{gray!60} - & \cellcolor{red!4.674617196702005}  -5.61 & \cellcolor{SoftGreen!9.893992932862119}  -4.51 & \cellcolor{SoftGreen!0.17667844522966902}  -4.99 \\
Group 3 & \cellcolor{SoftGreen!7.1015106593253705}  -4.64 & \cellcolor{red!6.988564167725539}  -8.39 & \cellcolor{gray!60} - & \cellcolor{red!6.141465480728503}  -7.37 & \cellcolor{red!9.718104381382652}  -11.66 \\
Group 4 & \cellcolor{SoftGreen!6.197139781639095}  -4.69 & \cellcolor{red!5.9203444564047425}  -7.10 & \cellcolor{red!6.163821825823982}  -7.40 & \cellcolor{gray!60} - & \cellcolor{red!8.02193859244451}  -9.63 \\
Group 5 & \cellcolor{SoftGreen!14.294750158127727}  -4.29 & \cellcolor{red!4.849251528568416}  -5.82 & \cellcolor{red!5.073265865485981}  -6.09 & \cellcolor{SoftGreen!24.09867172675527}  -3.80 & \cellcolor{gray!60} - \\\hline
\multicolumn{6}{l}{}\\[2pt]

\multicolumn{6}{l}{{\bf Taxonomy Domains}} \\\hline
\diagbox[width=6em]{Task 1}{Task 2} 
& Animals & Food & Interior & Sports & Transport \\\hline
Animals & \cellcolor{gray!60} - & \cellcolor{red!6.719879929048997}  -8.06 & \cellcolor{SoftGreen!27.41165234001882}  -3.63 & \cellcolor{red!4.866512029835813}  -5.84 & \cellcolor{SoftGreen!12.948560513030117}  -4.35 \\
Food & \cellcolor{red!13.65079365079365}  -16.38 & \cellcolor{gray!60} - & \cellcolor{SoftGreen!14.285714285714192}  -4.29 & \cellcolor{red!14.232804232804229}  -17.08 & \cellcolor{red!9.947089947089951}  -11.94 \\
Interior & \cellcolor{red!4.788538155072028}  -5.75 & \cellcolor{red!4.328838492185116}  -5.19 & \cellcolor{gray!60} - & \cellcolor{red!6.3591786699356465}  -7.63 & \cellcolor{SoftGreen!43.30370824394703}  -2.83 \\
Sports & \cellcolor{red!9.689658306070521}  -11.63 & \cellcolor{red!15.164940021810253}  -18.20 & \cellcolor{red!7.997091966557623}  -9.60 & \cellcolor{gray!60} - & \cellcolor{red!7.894856415848786}  -9.47 \\
Transport & \cellcolor{SoftGreen!16.27906976744177}  -4.19 & \cellcolor{red!7.067183462532299}  -8.48 & \cellcolor{SoftGreen!47.596899224806265}  -2.62 & \cellcolor{SoftGreen!26.511627906976827}  -3.67 & \cellcolor{gray!60} - \\\hline
\multicolumn{6}{l}{}\\[2pt]
    
\multicolumn{6}{l}{{\bf Question Types}} \\\hline
\diagbox[width=6em]{Task 1}{Task 2}
& Action & Color & Count & Scene & Subcat. \\\hline
Action & \cellcolor{gray!60} - & \cellcolor{red!57.00123915737299}  -68.40 & \cellcolor{red!75.37495192923984}  -90.45 & \cellcolor{red!16.32269367175149}  -19.59 & \cellcolor{red!10.479425714651969}  -12.58 \\
Color & \cellcolor{red!74.07521705139284}  -88.89 & \cellcolor{gray!60} - & \cellcolor{red!83.04530304900631}  -99.65 & \cellcolor{red!23.12471711311361}  -27.75 & \cellcolor{red!52.05118709624326}  -62.46 \\
Count & \cellcolor{red!82.64429312581063}  -99.17 & \cellcolor{red!83.063121487246}  -99.68 & \cellcolor{gray!60} - & \cellcolor{red!81.26621271076525}  -97.52 & \cellcolor{red!72.4978383052313}  -87.00 \\
Scene & \cellcolor{red!9.091433848995615}  -10.91 & \cellcolor{red!28.66928346032479}  -34.40 & \cellcolor{red!64.7752636034788}  -77.73 & \cellcolor{gray!60} - & \cellcolor{red!12.679904563996002}  -15.22 \\
Subcat. & \cellcolor{red!26.44189628615437}  -31.73 & \cellcolor{red!71.21056534885048}  -85.45 & \cellcolor{red!80.12436533744082}  -96.15 & \cellcolor{red!25.457812767414}  -30.55 & \cellcolor{gray!60} - \\\hline
\multicolumn{6}{l}{}\\[2pt]
\end{tabular}}
    \caption{Task difficulty measured by forgetting in pairwise tasks.
 Non-diagonal elements show relative accuracy drop (\%) after finetuning on Task 2.}
    \label{tab:Tpairs}
\end{table}
\section{Settings for Continual VQA} \label{sec:settings}
\subsection{Problem formulation}
In continual learning, model parameters $\bm{\theta}$ are incrementally updated as new data become available.
We assume that samples from tasks $t=1\dots T$ arrive sequentially as $D_t=\{\bm{x}_i, \bm{y}_i\}_{i=1}^{N_t}$, where $N_t$ is the number of data for task $t$.
Following previous work, VQA is formulated as a multi-label classification problem with soft targets $\bm{y}_i$~\cite{Anderson2017up-down}.
Starting from parameters $\bm{\theta}_{t-1}$ of the previous model, the updated parameters $\bm{\theta}_{t}$ are obtained by training on the new data $D_t$. 
Some approaches also use a memory $M_t$ containing a subset of samples from previous tasks, e.g. $D_{1}, \dots, D_{t-1}$.
In our setup, all tasks share a common output head which is extended with new classes from each task.
This allows inference to be task-agnostic but creates a more challenging setting than multi-head learning where separate heads are learned for each task~\cite{hussain2021towards}.
At the end of the training sequence, the objective is to achieve strong performance across all tasks observed so far. 
This objective encloses two challenges: 1) minimizing catastrophic forgetting of tasks seen earlier in training, 2) facilitating positive transfer to improve performance on new tasks~\cite{HADSELL20201028embracing}. 

\subsubsection{Language Setting}\label{sec:q_splits}
We create a third setting {\em Question Types}, where each task corresponds to learning to answer a different category of questions.
We use a classification scheme developed by~\citet{whitehead2021separating}
to form a sequence of five tasks: 
Count, Color, Scene-level, Subcategory, and Action recognition. The splits for Count, Color, and Subcategory questions are obtained from~\citet{whitehead2021separating}. We create two additional tasks from the remaining questions.
In particular, we cluster question embeddings from Sentence-BERT~\cite{reimers-2019-sentence-bert} \footnote{We use the `all-MiniLM-L6-v2' model and Fast Clustering algorithm from the sentence-transformers package (\url{https://www.sbert.net/}).} so that each cluster has at least 15 questions and a minimum cosine similarity of 0.8 between all embeddings.
We annotate clusters as `scene', `action' or `irrelevant' question types.
Based on a seed of 10K annotated questions, we retrieve all other questions with similarity above 0.8 and label them using the K-nearest neighbor algorithm ($K=5$).  
Question Types have a total of 140K train, 35K validation and 84K test samples (cf.\ Table~\ref{tab:stats}). 
Common question words and answers per task are presented in the Appendix (Figure~\ref{fig:top_ans_q}). 

We also experiment with a baseline Pseudo-Replay method for the Question Types setting.
Instead of storing raw data from previous tasks, we use a data augmentation method, inspired by~\cite{kafle2017data,kil2021discovering}. 
When training on task $t$, we augment the data $D_t$ by retrieving past questions based on their shared detected objects classes. For example, if an elephant is detected on the current picture, we retrieve a past question about an elephant. We then use the previous model $f_{\theta_{t-1}}$ to generate a distribution $\tilde{ \bm{y}} = f_{\theta_{t-1}}(\tilde{\bm{x}})$ which serves as soft targets for the new sample $\tilde{\bm{x}}$.
By not storing the original answers, we address privacy and efficiency concerns of replay approaches~\cite{van2018generative,Delangesurvey}.

\subsection{Evaluation Metrics}
After training on task $t$, we compute the VQA accuracy $A_{t,i}$ on data from the previous task $i$. 
We report the macro-average accuracy at the end of the training sequence: $\mathrm{A} = \frac{1}{T} \sum_{i=1}^T A_{T,i}$.
Following~\citet{riemer2018learning}, we report the learned accuracy $\mathrm{LA} = \frac{1}{T} \sum_{i=1}^T A_{i,i}$, which measures the ability to learn the new task $i$.
We also compute backward transfer $\mathrm{BWT} = \frac{1}{T-1} \sum_{i=1}^{T-1} A_{T,i} - A_{i,i}$~\cite{lopez2017GEM}, that captures the impact of catastrophic forgetting.

\begin{equation}
   S_{T,i} = \frac{1}{N} \sum_{j=1}^{N} (1-\mathrm{cos}(\bm{e_{Tj}}, \bm{e_{ij}}))\cdot \Delta^{Ti}_j
\end{equation}

\section{Results}
\subsection{Main Results}
\paragraph{Task Settings.}
Table~\ref{tab:results} summarizes the results averaged over five task orders using the UNITER backbone.
The results show an increasing difficulty for the three incremental learning task definitions, i.e. Diversity Domains $<$ Taxonomy Domains $<$ Question Types, which we will further investigate in Section \ref{sec:task_relationships}. 

\paragraph{Experimental Setup.} 
We first look into pairwise task relationships following studies in transfer~\cite{zamir2018taskonomy} and multitask learning ~\cite{pmlr-v119-standley20a, lu202012}.
In particular, we measure the extent to which each task is forgotten after training on a second task. 
We finetune a pretrained model on Task $T_{1}$ and compute the accuracy $A_{11}$ on its test set. Then, we finetune this model on another Task $T_{2}$ and compute the new accuracy $A_{12}$ on the test set of $T_{1}$.
Forgetting is measured as the relative accuracy drop: $(A_{12} - A_{11}) / A_{11}$.
Given the varying dataset sizes, we finetune on $T_{2}$ for a fixed number of 400 steps using a batch size of 512 and learning rate 5e-5.

\paragraph{Results.} 
Table~\ref{tab:Tpairs} shows the relative accuracy drop for all task pairs. 
Overall, we observe that each setting has a distinct pattern. 
Question Types is evidently a more challenging setting, where several task combinations show more than $90\%$ drop.
When comparing the visual settings, forgetting in Diverse Domains fluctuates less depending on the task pairing. 
This suggests that the task relationships in Taxonomy Domains might play a more important factor.
Although some relations make sense based on the expected similarity of the visual scenes, e.g., low forgetting between Food and Interior, others are less intuitive, e.g., low forgetting between Transport and Interior.
Moreover, certain second tasks seem to consistently affect the amount of forgetting after finetuning on them. 
Based on the total number of classes per task as shown in Table ~\ref{tab:stats}, we notice that the model is more robust against forgetting when Task $T_2$ has a wide range of possible answers (e.g., Interior); while $T_2$ with a narrow answer set (e.g., Food, Color, Count) lead to maximum forgetting.
{"value": "-6.58", "type": "Other"} {"value": "-5.21", "type": "Other"} {"value": "-4.84", "type": "Other"} {"value": "-7.09", "type": "Other"} {"value": "-4.55", "type": "Other"} {"value": "-5.61", "type": "Other"} {"value": "-4.51", "type": "Other"} {"value": "-4.99", "type": "Other"} {"value": "-4.64", "type": "Other"} {"value": "-8.39", "type": "Other"} {"value": "-7.37", "type": "Other"} {"value": "-11.66", "type": "Other"} {"value": "-4.69", "type": "Other"} {"value": "-7.10", "type": "Other"} {"value": "-7.40", "type": "Other"} {"value": "-9.63", "type": "Other"} {"value": "-4.29", "type": "Other"} {"value": "-5.82", "type": "Other"} {"value": "-6.09", "type": "Other"} {"value": "-3.80", "type": "Other"} {"value": "-8.06", "type": "Other"} {"value": "-3.63", "type": "Other"} {"value": "-5.84", "type": "Other"} {"value": "-4.35", "type": "Other"} {"value": "-16.38", "type": "Other"} {"value": "-4.29", "type": "Other"} {"value": "-17.08", "type": "Other"} {"value": "-11.94", "type": "Other"} {"value": "-5.75", "type": "Other"} {"value": "-5.19", "type": "Other"} {"value": "-7.63", "type": "Other"} {"value": "-2.83", "type": "Other"} {"value": "-11.63", "type": "Other"} {"value": "-18.20", "type": "Other"} {"value": "-9.60", "type": "Other"} {"value": "-9.47", "type": "Other"} {"value": "-4.19", "type": "Other"} {"value": "-8.48", "type": "Other"} {"value": "-2.62", "type": "Other"} {"value": "-3.67", "type": "Other"} {"value": "-68.40", "type": "Other"} {"value": "-90.45", "type": "Other"} {"value": "-19.59", "type": "Other"} {"value": "-12.58", "type": "Other"} {"value": "-88.89", "type": "Other"} {"value": "-99.65", "type": "Other"} {"value": "-27.75", "type": "Other"} {"value": "-62.46", "type": "Other"} {"value": "-99.17", "type": "Other"} {"value": "-99.68", "type": "Other"} {"value": "-97.52", "type": "Other"} {"value": "-87.00", "type": "Other"} {"value": "-10.91", "type": "Other"} {"value": "-34.40", "type": "Other"} {"value": "-77.73", "type": "Other"} {"value": "-15.22", "type": "Other"} {"value": "-31.73", "type": "Other"} {"value": "-85.45", "type": "Other"} {"value": "-96.15", "type": "Other"} {"value": "-30.55", "type": "Other"}