@article{ART002827495},
author={Hyeyeon Chung and CHOI JISOO and Heo TakSung and SEOSOOYOUNG},
title={Significance of Recall in Automatic Metrics for HT Evaluation},
journal={The Journal of Translation Studies},
issn={1229-795X},
year={2022},
volume={23},
number={1},
pages={81-100},
doi={10.15749/jts.2022.23.1.003}
TY - JOUR
AU - Hyeyeon Chung
AU - CHOI JISOO
AU - Heo TakSung
AU - SEOSOOYOUNG
TI - Significance of Recall in Automatic Metrics for HT Evaluation
JO - The Journal of Translation Studies
PY - 2022
VL - 23
IS - 1
PB - The Korean Association for Translation Studies
SP - 81
EP - 100
SN - 1229-795X
AB - In the automatic evaluation of translations, precision and recall are two indices that show how precisely (precision) and how much (recall) the system is able to recognize the well-translated portion in a translation. It would be ideal if two indices could be equally weighted in the evaluation system, since both accuracy and completeness are important criteria in evaluation of human translations (HT). This is, however, not easy, as both indices are negatively correlated. Papineni et al. (2002), for example, opted for precision, while Lavie et al. (2005) used both indices, giving recall nine times more weight than precision. The aim of this work is to examine which of the two indices correlates better with evaluation of professional evaluators and how much weight should be given each to precision and to recall. For this purpose, 459 translated texts were rated with precision, recall, F1 (harmonic mean of precision and recall) and Fmean (nine times higher weight on recall) as well as by professional evaluators. The results show that recall correlates better with human evaluation than precision in almost all cases, but not Fmean than F1, which were equivalent in all but one case. They indicate that recall is indeed a more important metric, but the weight as high as nine on recall is not ideal for HT evaluation.
KW - automatic evaluation;translation quality;precision;recall;F1;Fmean
DO - 10.15749/jts.2022.23.1.003
ER -
Hyeyeon Chung, CHOI JISOO, Heo TakSung and SEOSOOYOUNG. (2022). Significance of Recall in Automatic Metrics for HT Evaluation. The Journal of Translation Studies, 23(1), 81-100.
Hyeyeon Chung, CHOI JISOO, Heo TakSung and SEOSOOYOUNG. 2022, "Significance of Recall in Automatic Metrics for HT Evaluation", The Journal of Translation Studies, vol.23, no.1 pp.81-100. Available from: doi:10.15749/jts.2022.23.1.003
Hyeyeon Chung, CHOI JISOO, Heo TakSung, SEOSOOYOUNG "Significance of Recall in Automatic Metrics for HT Evaluation" The Journal of Translation Studies 23.1 pp.81-100 (2022) : 81.
Hyeyeon Chung, CHOI JISOO, Heo TakSung, SEOSOOYOUNG. Significance of Recall in Automatic Metrics for HT Evaluation. 2022; 23(1), 81-100. Available from: doi:10.15749/jts.2022.23.1.003
Hyeyeon Chung, CHOI JISOO, Heo TakSung and SEOSOOYOUNG. "Significance of Recall in Automatic Metrics for HT Evaluation" The Journal of Translation Studies 23, no.1 (2022) : 81-100.doi: 10.15749/jts.2022.23.1.003
Hyeyeon Chung; CHOI JISOO; Heo TakSung; SEOSOOYOUNG. Significance of Recall in Automatic Metrics for HT Evaluation. The Journal of Translation Studies, 23(1), 81-100. doi: 10.15749/jts.2022.23.1.003
Hyeyeon Chung; CHOI JISOO; Heo TakSung; SEOSOOYOUNG. Significance of Recall in Automatic Metrics for HT Evaluation. The Journal of Translation Studies. 2022; 23(1) 81-100. doi: 10.15749/jts.2022.23.1.003
Hyeyeon Chung, CHOI JISOO, Heo TakSung, SEOSOOYOUNG. Significance of Recall in Automatic Metrics for HT Evaluation. 2022; 23(1), 81-100. Available from: doi:10.15749/jts.2022.23.1.003
Hyeyeon Chung, CHOI JISOO, Heo TakSung and SEOSOOYOUNG. "Significance of Recall in Automatic Metrics for HT Evaluation" The Journal of Translation Studies 23, no.1 (2022) : 81-100.doi: 10.15749/jts.2022.23.1.003
