@article{ART002617136},
author={Hyunwoo Kim},
title={Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study},
journal={Modern English Education},
issn={1598-0782},
year={2020},
volume={21},
number={3},
pages={28-36},
doi={10.18095/meeso.2020.21.3.28}
TY - JOUR
AU - Hyunwoo Kim
TI - Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study
JO - Modern English Education
PY - 2020
VL - 21
IS - 3
PB - The Modern English Education Society
SP - 28
EP - 36
SN - 1598-0782
AB - Previous studies on rater variability have shown that the amount of attention directed towards each rating criterion was associated with rating criteria order in analytic rating scales. Thus, this study attempts to explore whether rating criteria order affects the amount of attention directed to rating criteria using think-aloud protocols. To achieve a research goal, 11 novice Korean raters rated two essays in two different rating criteria orders. In the standard-order rating rubric, the rating criteria of textual aspects of the essays, including Content and Organization, were first evaluated. Then, the rating criteria of grammatical aspects of the essays, including Vocabulary and Language use, were evaluated. This rating order was precisely reversed in the reverse-order rating rubric. The overall results of this study show that heightened attention from the raters was directed toward either the first- or last-presented rating criterion depending on the quality of each essay. Specifically, a differential amount of attention was directed towards Content and Language Use when rating a poor-quality essay. A significant implication of the study is that rating criteria order should be considered in developing analytic rating scales, and in training newly recruited raters.
KW - Rater criteria order/Think-aloud protocol/L2 writing assessment
DO - 10.18095/meeso.2020.21.3.28
ER -
Hyunwoo Kim. (2020). Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study. Modern English Education, 21(3), 28-36.
Hyunwoo Kim. 2020, "Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study", Modern English Education, vol.21, no.3 pp.28-36. Available from: doi:10.18095/meeso.2020.21.3.28
Hyunwoo Kim "Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study" Modern English Education 21.3 pp.28-36 (2020) : 28.
Hyunwoo Kim. Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study. 2020; 21(3), 28-36. Available from: doi:10.18095/meeso.2020.21.3.28
Hyunwoo Kim. "Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study" Modern English Education 21, no.3 (2020) : 28-36.doi: 10.18095/meeso.2020.21.3.28
Hyunwoo Kim. Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study. Modern English Education, 21(3), 28-36. doi: 10.18095/meeso.2020.21.3.28
Hyunwoo Kim. Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study. Modern English Education. 2020; 21(3) 28-36. doi: 10.18095/meeso.2020.21.3.28
Hyunwoo Kim. Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study. 2020; 21(3), 28-36. Available from: doi:10.18095/meeso.2020.21.3.28
Hyunwoo Kim. "Effects of Rating Criteria Order on Novice Korean Raters’ Attentional Behaviors in L2 Writing Assessment: A Think-Aloud Protocol Study" Modern English Education 21, no.3 (2020) : 28-36.doi: 10.18095/meeso.2020.21.3.28
