\begin{table*}[h]
\centering
\small
\begin{tabular}{lp{2.7cm}p{2.7cm}ccc}
\toprule 
  & \textbf{Premise} & \textbf{Hypothesis} & \textbf{EL2N} & \textbf{Gold Label} & \textbf{Prediction}\\
\midrule 
\multirow{10}{*}{\rotatebox{90}{Noise / Hard}}
& Social insurance taxes and contributions paid by Federal employees (575)	
& There are no taxes for the Federal employees.	
& 1.410452
& Entailment (0)
& Contradiction (2)
\\\cmidrule[0.01em](lr){2-6}
& um-hum um-hum yep you were very fortunate
& You were very unfortunate.	
& 1.410408
& Entailment (0)
& Contradiction (2)
\\\cmidrule[0.01em](lr){2-6}
& "Everyone is chanting for you," Nema told him.
& Everyone was silent.
& 1.410146
& Neutral (1)
& Contradiction (2)
\\\midrule[0.03em]
\multirow{7}{*}{\rotatebox{90}{Easy}}
& Many of them did.	
& None of them did.	
& 0.002198
& Contradiction (2)
& Contradiction (2)
\\\cmidrule[0.01em](lr){2-6}
& Yes ”doubly careful." He turned to me abruptly.	
& No, not careful at all. He slunk away.	
& 0.002147
& Contradiction (2)
& Contradiction (2)
\\\cmidrule[0.01em](lr){2-6}
& Many others exist, too.
& There are no others who exist.
& 0.002134
& Contradiction (2)
& Contradiction (2)
 \\
\bottomrule
\end{tabular}
\caption{
Examples from MNLI belonging to different score regions of EL2N. The highest scoring instances are mostly noisy samples, while the least scoring instances are very easy to learn.
} 
\label{tab:examples-mnli}
\end{table*}
\section{Background}
In this section, we describe the two metrics introduced by \citet{NEURIPS2021_ac56f8fe_datadiet} for pruning the training data: \textbf{GraNd} and its estimated variant, \textbf{EL2N}. 
\subsection{GraNd}
Consider $\mathbf{X} = \{x_i, y_i\}_{i=1}^N$ to be a training dataset for a given classification task with $K$ classes, where $x_i$ is the input (i.e. a sequence of tokens) and $y_i$ is its corresponding label. 
To estimate the importance of each training sample $(x_i, y_i)$,  \citet{NEURIPS2021_ac56f8fe_datadiet} propose utilizing the expected value of the loss gradient norm denoted as GraNd:

\begin{equation}
\label{eq:grand}
    \text{GraNd}(x_i, y_i) = \mathbb{E}_{\boldsymbol{w}}\left\|\boldsymbol{g}(x_i, y_i)\right\|_2
\end{equation}

\subsection{EL2N}
By defining $\psi^{(k)}(x_i) = \nabla_{\boldsymbol{w}}f^{(k)}(x_i)$ for the $k^\text{th}$ logit,
the loss gradient ($\boldsymbol{g}$) can be written as:

\begin{equation}
    \boldsymbol{g}(x_i, y_i) = \sum_{k=1}^{K}\nabla_{f^{(k)}}\mathcal{L}(f(x_i), y_i)^T\psi^{(k)}(x_i)
\end{equation}

Since the $\mathcal{L}(f(x_i), y_i)$ is a cross entropy loss, $\nabla_{f^{(k)}}\mathcal{L}(f(x_i), y_i)^T=p(x_i)^{(k)} - y^{(k)}_i$ , where $p(x_i)$ is the output probability vector of the model. By assuming orthogonality and uniform sizes across logits for $\psi$ over the training examples, the norm in Eq. \ref{eq:grand} is approximately equal to $\left\|p(x_i) - \boldsymbol{y}_i\right\|_2$ ($\boldsymbol{y}_i$ is the one-hot vector of the true label). As a result, an estimate of GraND is \textbf{EL2N}, which is defined as follows:

\begin{equation}
\label{eq:el2n}
    \text{EL2N}(x_i, y_i) = \mathbb{E}_{\boldsymbol{w}}\left\|p(x_i) - \boldsymbol{y}_i\right\|_2
\end{equation}

\section{Experiments}
In this section, we verify the effectiveness of GraNd and EL2N \citep{NEURIPS2021_ac56f8fe_datadiet} in the NLP domain. 
Our experimental setup is similar to \citet{NEURIPS2021_ac56f8fe_datadiet} for the followings:
(1) models are trained on the subsets of the data with higher GraNd and EL2N scores;
(2) based on the expectation over multiple weight initializations mentioned in Eq.~\ref{eq:grand} and Eq.~\ref{eq:el2n}, we average the scores over five independent training runs.\footnote{It is ten independent training runs in \citet{NEURIPS2021_ac56f8fe_datadiet}}; and,
(3) for putting our results in context, we utilized random pruning as the baseline\footnote{Random selection with the same proportion} and the accuracy of training on the whole dataset with no pruning as the target performance.

\section{Appendix}
\subsection{Dataset samples and their scores}
Examples with the highest and lowest EL2N scores are provided in Table~\ref{tab:examples-agnews} and Table~\ref{tab:examples-mnli} for AG News and MNLI datasets. 
{"value": "1.410452", "type": "Other"} {"value": "1.410408", "type": "Other"} {"value": "1.410146", "type": "Other"} {"value": "0.002198", "type": "Other"} {"value": "0.002147", "type": "Other"} {"value": "0.002134", "type": "Other"}