@article{ART003305935},
author={Jong Wook Kim},
title={ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs},
journal={Journal of The Korea Society of Computer and Information},
issn={1598-849X},
year={2026},
volume={31},
number={2},
pages={65-73},
doi={10.9708/jksci.2026.31.02.065}
TY - JOUR
AU - Jong Wook Kim
TI - ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs
JO - Journal of The Korea Society of Computer and Information
PY - 2026
VL - 31
IS - 2
PB - The Korean Society Of Computer And Information
SP - 65
EP - 73
SN - 1598-849X
AB - Large language models (LLMs) are widely used in natural language processing applications such as classification, summarization, and question answering. However, commercial LLMs are typically provided as black-box APIs, making it difficult to interpret the causes of their outputs or to quantitatively assess their reliability. In particular, existing approaches fail to provide a probabilistic characterization of how often output changes occur in response to input variations, or how consistently such changes arise. To address this limitation, this paper proposes ProbCert, a framework for estimating and certifying output change probabilities under input perturbations in black-box LLM settings. ProbCert repeatedly generates semantically valid input variations, observes whether output changes occur, and estimates the corresponding change probability, while continuing queries until a user-specified confidence level and error tolerance are satisfied. The framework integrates multiple confidence interval estimation methods, including the Wilson score interval, Empirical Bernstein bound, and the Clopper–Pearson interval, enabling systematic comparison of estimation accuracy and query efficiency under a unified procedure. Experimental results on both classification and generation tasks demonstrate that all variants of ProbCert reliably satisfy the specified confidence and error requirements. In particular, the Wilson score–based variant achieves certification with the fewest LLM queries, highlighting its practical efficiency in commercial LLM environments.
KW - Output change probability;Explainable artificial intelligence;Large language model
DO - 10.9708/jksci.2026.31.02.065
ER -
Jong Wook Kim. (2026). ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs. Journal of The Korea Society of Computer and Information, 31(2), 65-73.
Jong Wook Kim. 2026, "ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs", Journal of The Korea Society of Computer and Information, vol.31, no.2 pp.65-73. Available from: doi:10.9708/jksci.2026.31.02.065
Jong Wook Kim "ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs" Journal of The Korea Society of Computer and Information 31.2 pp.65-73 (2026) : 65.
Jong Wook Kim. ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs. 2026; 31(2), 65-73. Available from: doi:10.9708/jksci.2026.31.02.065
Jong Wook Kim. "ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs" Journal of The Korea Society of Computer and Information 31, no.2 (2026) : 65-73.doi: 10.9708/jksci.2026.31.02.065
Jong Wook Kim. ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs. Journal of The Korea Society of Computer and Information, 31(2), 65-73. doi: 10.9708/jksci.2026.31.02.065
Jong Wook Kim. ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs. Journal of The Korea Society of Computer and Information. 2026; 31(2) 65-73. doi: 10.9708/jksci.2026.31.02.065
Jong Wook Kim. ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs. 2026; 31(2), 65-73. Available from: doi:10.9708/jksci.2026.31.02.065
Jong Wook Kim. "ProbCert: Probabilistic Certification Framework for Black-box LLM Outputs" Journal of The Korea Society of Computer and Information 31, no.2 (2026) : 65-73.doi: 10.9708/jksci.2026.31.02.065
