\begin{table*}
\centering
\footnotesize
\begin{tabular}{lcccccc|c}
\toprule
& \multicolumn{2}{c}{\bf Lexical} & \multicolumn{2}{c}{\bf Entities} & \multicolumn{2}{c}{\bf Random} & \bf $\Delta$FRMT \\
\bf Model & pt-BR & pt-PT & pt-BR & pt-PT & pt-BR & pt-PT & pt\\\midrule
UR & 8.4 (2.7) & \scalebox{0.7}[1.0]{$-$}7.7 (\scalebox{0.7}[1.0]{$-$}2.5) & 6.9 (1.4) & \scalebox{0.7}[1.0]{$-$}6.3 (\scalebox{0.7}[1.0]{$-$}1.3) & 5.2 (0.3) & \scalebox{0.7}[1.0]{$-$}4.3 (\scalebox{0.7}[1.0]{$-$}0.0) & 0.2 (0.1)\\
M4 UR & 13.9 (5.2) & \scalebox{0.7}[1.0]{$-$}13.0 (\scalebox{0.7}[1.0]{$-$}5.0) & 8.2 (2.5) & \scalebox{0.7}[1.0]{$-$}7.9 (\scalebox{0.7}[1.0]{$-$}2.4) & 9.7 (2.9) & \scalebox{0.7}[1.0]{$-$}9.3 (\scalebox{0.7}[1.0]{$-$}2.8) & 0.2 (0.1)\\
M4 Prompts & 19.5 (5.7) & \scalebox{0.7}[1.0]{$-$}13.6 (\scalebox{0.7}[1.0]{$-$}3.2) & 10.5 (2.9) & \scalebox{0.7}[1.0]{$-$}7.8 (\scalebox{0.7}[1.0]{$-$}2.3) & 15.6 (3.4) & \scalebox{0.7}[1.0]{$-$}12.6 (\scalebox{0.7}[1.0]{$-$}2.7) & 1.9 (0.6)\\
M4 Prompts FT & 14.8 (4.9) & \scalebox{0.7}[1.0]{$-$}9.8 (\scalebox{0.7}[1.0]{$-$}2.8) & 8.7 (2.4) & \scalebox{0.7}[1.0]{$-$}6.4 (\scalebox{0.7}[1.0]{$-$}2.0) & 11.8 (2.8) & \scalebox{0.7}[1.0]{$-$}9.2 (\scalebox{0.7}[1.0]{$-$}2.2) & 1.6 (0.5)\\
PaLM 8B & 13.6 (5.1) & \scalebox{0.7}[1.0]{$-$}5.4 (\scalebox{0.7}[1.0]{$-$}1.8) & 8.6 (2.7) & \scalebox{0.7}[1.0]{$-$}3.3 (\scalebox{0.7}[1.0]{$-$}1.6) & 7.6 (1.7) & \scalebox{0.7}[1.0]{$-$}2.9 (\scalebox{0.7}[1.0]{$-$}0.7) & 2.8 (0.9)\\
PaLM 62B & 18.0 (6.2) & 0.3 (0.5) & 11.5 (3.4) & 0.3 (\scalebox{0.7}[1.0]{$-$}0.6) & 11.5 (2.5) & \scalebox{0.7}[1.0]{$-$}0.9 (\scalebox{0.7}[1.0]{$-$}0.4) & 6.5 (1.9)\\
PaLM 540B & 20.7 (6.4) & 0.2 (0.9) & 13.4 (3.7) & 0.2 (\scalebox{0.7}[1.0]{$-$}0.5) & 13.1 (2.9) & \scalebox{0.7}[1.0]{$-$}0.1 (0.0) & 7.7 (2.2)\\
Online & 20.6 (6.4) & \scalebox{0.7}[1.0]{$-$}20.6 (\scalebox{0.7}[1.0]{$-$}6.4) & 9.5 (2.8) & \scalebox{0.7}[1.0]{$-$}9.5 (\scalebox{0.7}[1.0]{$-$}2.8) & 22.3 (5.5) & \scalebox{0.7}[1.0]{$-$}22.3 (\scalebox{0.7}[1.0]{$-$}5.5) & 0.0 (0.0)\\
\midrule
& zh-CN & zh-TW & zh-CN & zh-TW & zh-CN & zh-TW & zh\\\midrule
UR & 4.4 (1.1) & \scalebox{0.7}[1.0]{$-$}3.4 (\scalebox{0.7}[1.0]{$-$}1.0) & 5.0 (1.8) & \scalebox{0.7}[1.0]{$-$}4.9 (\scalebox{0.7}[1.0]{$-$}1.8) & 3.8 (0.9) & \scalebox{0.7}[1.0]{$-$}3.0 (\scalebox{0.7}[1.0]{$-$}0.8) & 0.3 (0.0)\\
M4 UR & 14.5 (7.0) & \scalebox{0.7}[1.0]{$-$}14.4 (\scalebox{0.7}[1.0]{$-$}7.0) & 11.7 (2.5) & \scalebox{0.7}[1.0]{$-$}11.5 (\scalebox{0.7}[1.0]{$-$}2.5) & 10.0 (1.9) & \scalebox{0.7}[1.0]{$-$}10.1 (\scalebox{0.7}[1.0]{$-$}1.9) & 0.0 (0.0)\\
M4 Prompts & 14.4 (5.6) & \scalebox{0.7}[1.0]{$-$}14.4 (\scalebox{0.7}[1.0]{$-$}5.8) & 11.7 (2.6) & \scalebox{0.7}[1.0]{$-$}10.9 (\scalebox{0.7}[1.0]{$-$}2.7) & 10.7 (2.1) & \scalebox{0.7}[1.0]{$-$}10.1 (\scalebox{0.7}[1.0]{$-$}1.6) & 0.1 (0.0)\\
M4 Prompts FT & 14.2 (5.8) & \scalebox{0.7}[1.0]{$-$}13.5 (\scalebox{0.7}[1.0]{$-$}5.8) & 11.9 (2.7) & \scalebox{0.7}[1.0]{$-$}11.1 (\scalebox{0.7}[1.0]{$-$}2.6) & 9.9 (1.5) & \scalebox{0.7}[1.0]{$-$}9.6 (\scalebox{0.7}[1.0]{$-$}1.6) & 0.1 (\scalebox{0.7}[1.0]{$-$}0.0)\\
PaLM 8B & 6.2 (4.1) & \scalebox{0.7}[1.0]{$-$}2.3 (\scalebox{0.7}[1.0]{$-$}2.3) & 4.0 (1.0) & \scalebox{0.7}[1.0]{$-$}0.7 (\scalebox{0.7}[1.0]{$-$}0.9) & 4.5 (0.5) & \scalebox{0.7}[1.0]{$-$}1.0 (0.1) & 1.7 (0.4)\\
PaLM 62B & 12.3 (3.8) & \scalebox{0.7}[1.0]{$-$}2.9 (\scalebox{0.7}[1.0]{$-$}0.7) & 8.5 (1.8) & \scalebox{0.7}[1.0]{$-$}3.0 (\scalebox{0.7}[1.0]{$-$}1.0) & 8.7 (1.5) & \scalebox{0.7}[1.0]{$-$}2.6 (0.2) & 3.3 (0.9)\\
PaLM 540B & 15.0 (5.0) & \scalebox{0.7}[1.0]{$-$}0.5 (1.0) & 10.2 (2.2) & \scalebox{0.7}[1.0]{$-$}3.3 (\scalebox{0.7}[1.0]{$-$}0.8) & 9.9 (2.1) & \scalebox{0.7}[1.0]{$-$}1.4 (0.2) & 4.7 (1.6)\\
Online & 17.8 (6.2) & \scalebox{0.7}[1.0]{$-$}17.8 (\scalebox{0.7}[1.0]{$-$}6.2) & 13.4 (2.8) & \scalebox{0.7}[1.0]{$-$}13.4 (\scalebox{0.7}[1.0]{$-$}2.8) & 16.3 (3.3) & \scalebox{0.7}[1.0]{$-$}16.3 (\scalebox{0.7}[1.0]{$-$}3.3) & 0.0 (0.0)\\
\bottomrule
\end{tabular}
\caption{FRMT test set deltas between matched and mismatched references, shown in the format: $\Delta$BLEU~($\Delta$BLEURT)\@. Negative numbers in a minority region column indicate that the model achieved a higher score when evaluated with respect to the majority region, despite being asked to output for the minority region. The last column shows deltas between FRMT scores evaluated with respect to matched vs.~mismatched references.}
\label{tab:mismatched_references}
\end{table*}
Popular shared tasks have not included region-targeted translation either:
The Workshop on Machine Translation (WMT) has included translation between similar languages \cite[e.g.][]{akhbardeh-etal-2021-findings}, while the Workshop on NLP for Similar Languages, Varieties and Dialects (VarDial) focuses mainly on classification and not translation \cite[e.g.][]{vardial-2021-nlp}.

\textbf{BLEU \citep{papineni-etal-2002-bleu}:} This metric is based on token $n$-gram precision. We use \texttt{sacrebleu.corpus\_bleu} from \citet{post-2018-call}.

\textbf{chrF \citep{popovic-2015-chrf}:} This metric is based on F1 using character $n$-grams. We use \texttt{sacrebleu.corpus\_chrf} from \citet{post-2018-call}.

To derive a score that is more sensitive to the features of regional varieties, we filter MQM error categories to those most indicative of region mismatch.  Specifically, for each language (Portuguese and Mandarin), we train a logistic regression model over ratings of gold translations to predict whether the rater's region is matched (0) or mismatched (1) to the translator's region.  The model uses error types as features, and assigns positive coefficients to errors that are associated with region mismatch.  We designate features as ``mismatch predictive'' if they have both a positive coefficient and a significant $p$-value (<0.05).\footnote{We believe a priori that no error type should be robustly indicative of \emph{matching} region, as having a matched translator and rater should ideally result in no errors.}  Having identified these error types ($8$ in Portuguese, and $7$ in Mandarin), we run a second regression restricted to just these features.  We calculate ``Reweighted MQM'' (R-MQM) as the probability this model assigns to the mismatch category, giving scores in the range [$0$,$1$], as opposed to raw MQM scores which can cover [$0$,$\infty$] if arbitrarily many errors are found.

We calculate the metric over all model outputs for the \texttt{lexical} bucket, covering both regional outputs of a given language. For each term we calculate the number of sentences containing a matched variant and the number of sentences containing a mismatched variant; the sum of these over all terms are $N_{match}$ and $N_{mismatch}$, respectively. From this, we calculate the model's lexical accuracy (LA) score for the given language as:

Figure \ref{fig:mqm_scores} presents human evaluation metrics for our baseline models on the 25\% sample of our test set described in \S\ref{sec:translate_quality_metrics}. We first consider gold translations: in order to validate that our process for collecting reference translations and human ratings elicited the sought-after linguistic variance, we can examine whether translations by raters in one speaker group were consistently rated higher by other raters in that same group (the ``matched'' case) than by raters in the alternative speaker group (the ``mismatched'' case). Indeed, we do see that this is the case; on average, matched reference/rater pairs received an MQM penalty of $1.73$, which is significantly lower (better) than the MQM penalty of $3.55$ received on average by mismatched pairs, as measured by a paired $t$-test ($t=-3.34; p<0.001$). This effect is strongest in the \texttt{lexical} bucket, presumably due to the high rate of region-distinct terms in these sentences.\footnote{We note that ratings tend to be somewhat better in the \texttt{entity} bucket, across both matched and mismatched settings.  This could indicate that professional translators are less prone to errors when the source is ``culturally relevant'', e.g.~when translating sentences about an entity native to the area where the target language is spoken.}

For Mandarin (on the full 25\% sample), a large gap remains between expert translations and our baselines (Gold: $2.5$ vs.~PaLM: $8.8$).  Our results indicate that better region handling will be needed to close this gap; our baselines are not effectively leveraging region information (as evidenced by small match vs.~mismatch deltas), and even the best-case expert translation scores poorly in the mismatched region (Gold match: $2.5$ vs.~Gold mismatch: $5.2$).\footnote{Note, this is ignoring orthographic differences, as we transliterate outputs to the rater's script.  See section \S\ref{sec:script_analysis} for analysis of script differences.}

\subsection{Mismatched references}\label{mismatched_references}
All else being equal, a model that has perfectly solved the FRMT task should achieve higher performance when evaluated against a reference in the target region than a reference from a different region. To measure the extent to which this holds for our baseline models, we show the delta between matched and mismatched reference on the test set in Table \ref{tab:mismatched_references}.
{"value": "8.4", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "6.9", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "5.2", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "0.2", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "13.9", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "8.2", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "9.7", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "0.2", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "19.5", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "10.5", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "15.6", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "1.9", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "14.8", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "8.7", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "11.8", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "1.6", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "13.6", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "8.6", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "7.6", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "2.8", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "18.0", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "0.3", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical pt-PT", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "11.5", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "0.3", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities pt-PT", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "11.5", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "6.5", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "20.7", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "0.2", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical pt-PT", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "13.4", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "0.2", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities pt-PT", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "13.1", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "7.7", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "20.6", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Lexical pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "9.5", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Entities pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "22.3", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Random pt-BR", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "0.0", "type": "Result", "training data/set": "xx", "test data/set": "FRMT pt", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "4.4", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "5.0", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "3.8", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "0.3", "type": "Result", "training data/set": "mC4 OPUS", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "UR", "model settings": {"xx": "yy"}} {"value": "14.5", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "11.7", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "10.0", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "0.0", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-UR", "model settings": {"xx": "yy"}} {"value": "14.4", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "11.7", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "10.7", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "0.1", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts", "model settings": {"xx": "yy"}} {"value": "14.2", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "11.9", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "9.9", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "0.1", "type": "Result", "training data/set": "a mixture of monolingual and parallel data from 112 languages mined from the web", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "M4-Prompts FT", "model settings": {"xx": "yy"}} {"value": "6.2", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "4.0", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "4.5", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "1.7", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 8B", "model settings": {"xx": "yy"}} {"value": "12.3", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "8.5", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "8.7", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "3.3", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 62B", "model settings": {"xx": "yy"}} {"value": "15.0", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "10.2", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "9.9", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "4.7", "type": "Result", "training data/set": "a multilingual corpus", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "PaLM 540B", "model settings": {"xx": "yy"}} {"value": "17.8", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Lexical zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "13.4", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Entities zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "16.3", "type": "Result", "training data/set": "xx", "test data/set": "FRMT Random zh-CN", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}} {"value": "0.0", "type": "Result", "training data/set": "xx", "test data/set": "FRMT zh", "task": ["Few-Shot Region-Aware Machine Translation", "FRMT", "Machine Translation"], "metric": "delta BLEU", "experimental settings": {"xx": "yy"}, "model": "Online", "model settings": {"xx": "yy"}}