\begin{table*}[!t]
\centering
\label{tab-Experimental Results}
\setlength{\tabcolsep}{5.5mm}{
\begin{tabular}{@{}ccccccc@{}}
\toprule
                                    & \multicolumn{2}{c}{WordNet} & \multicolumn{2}{c}{Oxford} & \multicolumn{2}{c}{Urban} \\ \midrule
                                    & BLEU         & NIST         & BLEU         & NIST        & BLEU        & NIST        \\
\toprule
Ours                          & \bf{32.05}        & \bf{74.71}      & 27.10        & 79.42        & \bf{19.44}        & \bf{41.01}              \\
w/ $\rm Mean()$                          & 31.07        & 71.48      & \bf{27.13}        & \bf{80.33}        & 18.57        & 40.29             \\
w/ One-stage training                         & 31.75        & 73.79      & 27.06        & 79.90        &  16.49       &   31.46           \\
\bottomrule
T5-Base                           & 31.72        & 57.35        & 25.44        & 66.92        & 17.66        & 26.86              \\
\bottomrule
\end{tabular}}
\caption{Ablation study results on test sets. The best numbers are in bold.}
\label{ablation study}
\end{table*}
\subsection{Task Formulation}
\label{Task Formulation}
Given a word or phrase $W = \{w_i,...,w_j\}$ and its surrounding context $C = \{w_0,...,W_k\}(0<i<j<k)$, the task of definition generation is to generate the definition $D = \{d_0,...d_T\}$ to explain the meaning of $W$ under $C$. This process can be formulated as:

\begin{equation}\label{encoder}
\mathbf{H}_0={\rm Emb}({\rm Splice}(W, C))
\end{equation}

After encoding, we use a pooling function $\rm f()$ to aggregate the $\mathbf{H}_{target}$ and $\mathbf{G}^{L_D}$ respectively, and obtain {target word representation} $\mathbf{h}$ and {definition representation} $\mathbf{g}$ with the same length:

\begin{equation}\label{eq:contrastive}
L_C = \sum_{i=1}^N -log\frac{e^{{\rm sim}(h_i,g_i)}/\tau}{\sum_{j=1}^N e^{{\rm sim}(h_i,g_j)}/\tau}
\end{equation}

For evaluation, we follow previous works and acquire three popular datasets, which are ensembled by \citet{ishiwatari2019learning}\footnote{\url{http://www.tkl.iis.u-tokyo.ac.jp/~ishiwatari/naacl_data.zip}}.
Each entry in a dataset consists of three elements: (1) a target word or phrase, (2) the corresponding definition, and (3) one usage example of the target as a local context. If a target has multiple definitions and examples, we treat them as different entries. For fair comparison, each dataset is split into $\textit{train}$, $\textit{dev}$ and $\textit{test}$ sets according to \citet{ishiwatari2019learning}. The statistics of these datasets are shown in Table~\ref{Statistics of datasets}.

\subsection{Automatic Metrics}
Following common practice, we adopt two automatic evaluation metrics to assess the quality of the definitions generated by each model.
\paragraph{BLEU}
The metric BLEU \cite{papineni2002bleu} has been widely used in previous works to measure the closeness between the generated results and human reference. It measures the geometric average of the precision over hypothesis n-grams with an additional penalty to discourage short definition.
\paragraph{NIST}
NIST~\cite{doddington2002automatic} is similar to BLEU, but considers up-weighting rare, informative n-grams. We use NLTK\footnote{\url{https://www.nltk.org}} tool to calculate NIST metric.

\subsection{Experimental Setups}
\label{Settings}
We train all models in PyTorch\footnote{\url{https://github.com/pytorch/pytorch}} \cite{paszke2019pytorch}, and use the HuggingFace\footnote{\url{https://github.com/huggingface/}} \cite{wolf2019huggingface} implementation of T5. We train each model on a V100 GPU. For compared models, we replicate experiments following the implementations details released by \citet{huang2021definition}. For training our model, we use the base version of T5 with the same size of \citet{huang2021definition}. 
For each dataset, we finetune it using Adam ~\cite{kingma2014adam} optimizer with an initial learning rate of 3e-4 and the batch size of 16. In all the experiments, we train our model with a two-stage strategy as described in the previous section. Please refer to Appendix~\ref{Detailed Training Setting} for the detailed training settings in each stage, like max-epoch and early-stop threshold.

As shown in Table~\ref{ablation study}, replacing the pooling function $Max()$ with the mean-pooling $Mean()$ will bring in different changes on different datasets. Whereas the automatic scores drop a lot on WordNet and Urban datasets, they increase a bit on Oxford dataset. This indicates that the choice of pooling function might be empirically motivated, and in general the effect of contrastive learning does not vary a lot when the pooling function changes.
{"value": "32.05", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "74.71", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "27.10", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "79.42", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "19.44", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "41.01", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"xx": "yy"}} {"value": "31.07", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "71.48", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "27.13", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "80.33", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "18.57", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "40.29", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ Mean()": "true"}} {"value": "31.75", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "73.79", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "27.06", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "79.90", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "16.49", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "31.46", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": ["T5-Contrast", "Ours"], "model settings": {"w/ One-stage training": "true"}} {"value": "31.72", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}} {"value": "57.35", "type": "Result", "training data/set": "WordNet", "test data/set": "WordNet", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}} {"value": "25.44", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}} {"value": "66.92", "type": "Result", "training data/set": "Oxford", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}} {"value": "17.66", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "BLEU", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}} {"value": "26.86", "type": "Result", "training data/set": "Urban", "test data/set": "Oxford", "task": "Definition Generation", "metric": "NIST", "experimental settings": {"xx": "yy"}, "model": "T5-Base", "model settings": {"w/ One-stage training": "true"}}