@article{ART003338856},
author={Jiman Cha and Choong-hee Cho},
title={End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines},
journal={Journal of The Korea Society of Computer and Information},
issn={1598-849X},
year={2026},
volume={31},
number={5},
pages={11-27}
TY - JOUR
AU - Jiman Cha
AU - Choong-hee Cho
TI - End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines
JO - Journal of The Korea Society of Computer and Information
PY - 2026
VL - 31
IS - 5
PB - The Korean Society Of Computer And Information
SP - 11
EP - 27
SN - 1598-849X
AB - Large-scale services widely use distributed pipelines for real-time log processing. However, default buffering policies, although intended to protect system resources, can create bottlenecks that degrade real-time performance. This study constructed a large-scale distributed load environment and tracked end-to-end latency from log generation to final data warehouse loading at the millisecond level.
Experiments were conducted across a five-stage optimization scenario by adjusting buffer sizes, wait times, and scan intervals. The results show that a low-latency configuration can reduce explicit buffering delay, but may increase packet-level overhead and degrade throughput. In contrast, the proposed hybrid configuration does not aim for absolute optimality in a single metric; instead, it applies cross-layer tuning to mitigate ingestion-layer traffic variability while minimizing downstream transmission delays. Under the evaluated conditions, the hybrid configuration achieved the lowest P95 latency, prevented pipeline collapse in micro-batch environments, and maintained throughput exceeding approximately 300 events per second.
KW - Data Pipeline;End-to-End Latency;Throughput;Distributed Systems;Real-time Processing;Micro-Batching
DO -
UR -
ER -
Jiman Cha and Choong-hee Cho. (2026). End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines. Journal of The Korea Society of Computer and Information, 31(5), 11-27.
Jiman Cha and Choong-hee Cho. 2026, "End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines", Journal of The Korea Society of Computer and Information, vol.31, no.5 pp.11-27.
Jiman Cha, Choong-hee Cho "End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines" Journal of The Korea Society of Computer and Information 31.5 pp.11-27 (2026) : 11.
Jiman Cha, Choong-hee Cho. End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines. 2026; 31(5), 11-27.
Jiman Cha and Choong-hee Cho. "End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines" Journal of The Korea Society of Computer and Information 31, no.5 (2026) : 11-27.
Jiman Cha; Choong-hee Cho. End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines. Journal of The Korea Society of Computer and Information, 31(5), 11-27.
Jiman Cha; Choong-hee Cho. End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines. Journal of The Korea Society of Computer and Information. 2026; 31(5) 11-27.
Jiman Cha, Choong-hee Cho. End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines. 2026; 31(5), 11-27.
Jiman Cha and Choong-hee Cho. "End-to-End Latency Optimization and Resource Trade-off Analysis in Large-Scale Real-Time Data Pipelines" Journal of The Korea Society of Computer and Information 31, no.5 (2026) : 11-27.
