@article{ART003063608},
author={Sion Min and Youyang Kim and Byungchul Tak},
title={Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment},
journal={Journal of The Korea Society of Computer and Information},
issn={1598-849X},
year={2024},
volume={29},
number={3},
pages={127-136},
doi={10.9708/jksci.2024.29.03.127}
TY - JOUR
AU - Sion Min
AU - Youyang Kim
AU - Byungchul Tak
TI - Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment
JO - Journal of The Korea Society of Computer and Information
PY - 2024
VL - 29
IS - 3
PB - The Korean Society Of Computer And Information
SP - 127
EP - 136
SN - 1598-849X
AB - This study explores the correlation between system anomalies and large-scale logs within the Spark cluster environment. While research on anomaly detection using logs is growing, there remains a limitation in adequately leveraging logs from various components of the cluster and considering the relationship between anomalies and the system. Therefore, this paper analyzes the distribution of normal and abnormal logs and explores the potential for anomaly detection based on the occurrence of log templates. By employing Hadoop and Spark, normal and abnormal log data are generated, and through t-SNE and K-means clustering, templates of abnormal logs in anomalous situations are identified to comprehend anomalies. Ultimately, unique log templates occurring only during abnormal situations are identified, thereby presenting the potential for anomaly detection.
KW - Log analysis;Anomaly detection;Distributed system;Spark;Hadoop;Log template
DO - 10.9708/jksci.2024.29.03.127
ER -
Sion Min, Youyang Kim and Byungchul Tak. (2024). Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment. Journal of The Korea Society of Computer and Information, 29(3), 127-136.
Sion Min, Youyang Kim and Byungchul Tak. 2024, "Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment", Journal of The Korea Society of Computer and Information, vol.29, no.3 pp.127-136. Available from: doi:10.9708/jksci.2024.29.03.127
Sion Min, Youyang Kim, Byungchul Tak "Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment" Journal of The Korea Society of Computer and Information 29.3 pp.127-136 (2024) : 127.
Sion Min, Youyang Kim, Byungchul Tak. Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment. 2024; 29(3), 127-136. Available from: doi:10.9708/jksci.2024.29.03.127
Sion Min, Youyang Kim and Byungchul Tak. "Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment" Journal of The Korea Society of Computer and Information 29, no.3 (2024) : 127-136.doi: 10.9708/jksci.2024.29.03.127
Sion Min; Youyang Kim; Byungchul Tak. Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment. Journal of The Korea Society of Computer and Information, 29(3), 127-136. doi: 10.9708/jksci.2024.29.03.127
Sion Min; Youyang Kim; Byungchul Tak. Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment. Journal of The Korea Society of Computer and Information. 2024; 29(3) 127-136. doi: 10.9708/jksci.2024.29.03.127
Sion Min, Youyang Kim, Byungchul Tak. Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment. 2024; 29(3), 127-136. Available from: doi:10.9708/jksci.2024.29.03.127
Sion Min, Youyang Kim and Byungchul Tak. "Anomalous Pattern Analysis of Large-Scale Logs with Spark Cluster Environment" Journal of The Korea Society of Computer and Information 29, no.3 (2024) : 127-136.doi: 10.9708/jksci.2024.29.03.127
