@article{ART003332476},
author={Jun-Yong Park and Gyun-Ho Kim and Kang-Rae Jo and KIM, TAEKOOK},
title={Implementation of a Multimodal LLM-based Psychological Counseling AI System},
journal={Journal of Internet of Things and Convergence},
issn={2466-0078},
year={2026},
volume={12},
number={2},
pages={173-181}
TY - JOUR
AU - Jun-Yong Park
AU - Gyun-Ho Kim
AU - Kang-Rae Jo
AU - KIM, TAEKOOK
TI - Implementation of a Multimodal LLM-based Psychological Counseling AI System
JO - Journal of Internet of Things and Convergence
PY - 2026
VL - 12
IS - 2
PB - The Korea Internet of Things Society
SP - 173
EP - 181
SN - 2466-0078
AB - This study proposes and empirically evaluates a multimodal emotion recognition-based AI psychological counseling system that integrates speech, text, and facial expressions, aiming to address the rapidly increasing demand for mental health services in modern society and the limitations of conventional text-based counseling chatbots in capturing nonverbal cues. From a technical perspective, Wav2Vec2 was employed for Korean speech recognition, KoBERT for textual context analysis, and ResNet18 for real-time facial expression recognition, each serving as the backbone model for its respective modality. The emotion probability distributions generated from each model were integrated using a decision-level late fusion approach. Experimental results demonstrate that the proposed system improves the consistency and accuracy of emotional state recognition compared to single-modality approaches, while also significantly enhancing the understanding of counseling context. In particular, the integrated emotion analysis combining speech, text, and facial expression data enables more precise reflection of users’ emotional changes during the counseling process, thereby confirming its potential as a supportive tool for assisting counselors in decision-making.
KW - Multimodal;Emotion recognition;Digital healthcare;Large Language Model (LLM);Affective computing;Artificial intelligence
DO -
UR -
ER -
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo and KIM, TAEKOOK. (2026). Implementation of a Multimodal LLM-based Psychological Counseling AI System. Journal of Internet of Things and Convergence, 12(2), 173-181.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo and KIM, TAEKOOK. 2026, "Implementation of a Multimodal LLM-based Psychological Counseling AI System", Journal of Internet of Things and Convergence, vol.12, no.2 pp.173-181.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo, KIM, TAEKOOK "Implementation of a Multimodal LLM-based Psychological Counseling AI System" Journal of Internet of Things and Convergence 12.2 pp.173-181 (2026) : 173.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo, KIM, TAEKOOK. Implementation of a Multimodal LLM-based Psychological Counseling AI System. 2026; 12(2), 173-181.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo and KIM, TAEKOOK. "Implementation of a Multimodal LLM-based Psychological Counseling AI System" Journal of Internet of Things and Convergence 12, no.2 (2026) : 173-181.
Jun-Yong Park; Gyun-Ho Kim; Kang-Rae Jo; KIM, TAEKOOK. Implementation of a Multimodal LLM-based Psychological Counseling AI System. Journal of Internet of Things and Convergence, 12(2), 173-181.
Jun-Yong Park; Gyun-Ho Kim; Kang-Rae Jo; KIM, TAEKOOK. Implementation of a Multimodal LLM-based Psychological Counseling AI System. Journal of Internet of Things and Convergence. 2026; 12(2) 173-181.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo, KIM, TAEKOOK. Implementation of a Multimodal LLM-based Psychological Counseling AI System. 2026; 12(2), 173-181.
Jun-Yong Park, Gyun-Ho Kim, Kang-Rae Jo and KIM, TAEKOOK. "Implementation of a Multimodal LLM-based Psychological Counseling AI System" Journal of Internet of Things and Convergence 12, no.2 (2026) : 173-181.
