Applying Reinforcement learning (RL) following maximum likelihood estimation (MLE) pre-training is a versatile method for enhancing neural machine translation (NMT) performance. However, recent work has argued that the gains produced by RL for NMT are mostly due to promoting tokens that have already received a fairly high probability in pre-training. We hypothesize that the large action space is a main obstacle to RL’s effectiveness in MT, and conduct two sets of experiments that lend support to our hypothesis. First, we find that reducing the size of the vocabulary improves RL’s effectiveness. Second, we find that effectively reducing the dimension of the action space without changing the vocabulary also yields notable improvement as evaluated by BLEU, semantic similarity, and human evaluation. Indeed, by initializing the network’s final fully connected layer (that maps the network’s internal dimension to the vocabulary dimension), with a layer that generalizes over similar actions, we obtain a substantial improvement in RL performance: 1.5 BLEU points on average.
@inproceedings{DBLP:conf/coling/YehudaiCFA22,
author = {Asaf Yehudai and
Leshem Choshen and
Lior Fox and
Omri Abend},
editor = {Nicoletta Calzolari and
Chu{-}Ren Huang and
Hansaem Kim and
James Pustejovsky and
Leo Wanner and
Key{-}Sun Choi and
Pum{-}Mo Ryu and
Hsin{-}Hsi Chen and
Lucia Donatelli and
Heng Ji and
Sadao Kurohashi and
Patrizia Paggio and
Nianwen Xue and
Seokhwan Kim and
Younggyun Hahm and
Zhong He and
Tony Kyungil Lee and
Enrico Santus and
Francis Bond and
Seung{-}Hoon Na},
title = {Reinforcement Learning with Large Action Spaces for Neural Machine
Translation},
booktitle = {Proceedings of the 29th International Conference on Computational
Linguistics, {COLING} 2022, Gyeongju, Republic of Korea, October 12-17,
2022},
pages = {4544--4556},
publisher = {International Committee on Computational Linguistics},
year = {2022},
url = {https://aclanthology.org/2022.coling-1.401},
timestamp = {Thu, 13 Oct 2022 17:29:38 +0200},
biburl = {https://dblp.org/rec/conf/coling/YehudaiCFA22.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
See the full reference list in the paper.