@article{ART003372487},
author={Yuna Choi and PARK JONG DO and Haein Lee and CHO JANE},
title={Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System},
journal={Journal of the Korean Society for Library and Information Science},
issn={1225-598X},
year={2026},
volume={60},
number={3},
pages={125-143},
doi={10.4275/KSLIS.2026.60.3.125}
TY - JOUR
AU - Yuna Choi
AU - PARK JONG DO
AU - Haein Lee
AU - CHO JANE
TI - Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System
JO - Journal of the Korean Society for Library and Information Science
PY - 2026
VL - 60
IS - 3
PB - 한국문헌정보학회
SP - 125
EP - 143
SN - 1225-598X
AB - The Retrieval-Augmented Generation structure is a technology that generates responses by referencing a knowledge base, thereby supporting reliable responses. However, the quality of answers from a RAG structure can vary depending on the search performance of the knowledge base. In particular, when the knowledge base consists of norms in various specific domains composed of structured attributes or domain specific vocabulary, it is difficult to assume that embedding-based methods will unconditionally guarantee superior performance. Thereby, this study created a knowledge base based on the rules and regulations of a university's archive. Then, two different systems using the same LLM were built, with two different retrieval methods, BM25 and embedding. And then, the study compared and evaluated how search methods can vary depending on the user's query type and the structure of the knowledge base. Experimental results showed that BM25 demonstrated superior performance direct-answer and error-premising queries, where exact keyword matching is crucial, while the embedding method showed superior performance in inference and term-variation queries.
KW - Generative AI;RAG (Retrieval-Augmented Generation);Records Retention Schedule;Embedding;BM25
DO - 10.4275/KSLIS.2026.60.3.125
ER -
Yuna Choi, PARK JONG DO, Haein Lee and CHO JANE. (2026). Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System. Journal of the Korean Society for Library and Information Science, 60(3), 125-143.
Yuna Choi, PARK JONG DO, Haein Lee and CHO JANE. 2026, "Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System", Journal of the Korean Society for Library and Information Science, vol.60, no.3 pp.125-143. Available from: doi:10.4275/KSLIS.2026.60.3.125
Yuna Choi, PARK JONG DO, Haein Lee, CHO JANE "Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System" Journal of the Korean Society for Library and Information Science 60.3 pp.125-143 (2026) : 125.
Yuna Choi, PARK JONG DO, Haein Lee, CHO JANE. Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System. 2026; 60(3), 125-143. Available from: doi:10.4275/KSLIS.2026.60.3.125
Yuna Choi, PARK JONG DO, Haein Lee and CHO JANE. "Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System" Journal of the Korean Society for Library and Information Science 60, no.3 (2026) : 125-143.doi: 10.4275/KSLIS.2026.60.3.125
Yuna Choi; PARK JONG DO; Haein Lee; CHO JANE. Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System. Journal of the Korean Society for Library and Information Science, 60(3), 125-143. doi: 10.4275/KSLIS.2026.60.3.125
Yuna Choi; PARK JONG DO; Haein Lee; CHO JANE. Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System. Journal of the Korean Society for Library and Information Science. 2026; 60(3) 125-143. doi: 10.4275/KSLIS.2026.60.3.125
Yuna Choi, PARK JONG DO, Haein Lee, CHO JANE. Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System. 2026; 60(3), 125-143. Available from: doi:10.4275/KSLIS.2026.60.3.125
Yuna Choi, PARK JONG DO, Haein Lee and CHO JANE. "Comparative Analysis of BM25 and Embedding Retrieval Performance by Question Type in a Records Schedule and Regulations-Based RAG(Retrieval-Augmented Generation) Question Answering System" Journal of the Korean Society for Library and Information Science 60, no.3 (2026) : 125-143.doi: 10.4275/KSLIS.2026.60.3.125
