@article{ART003104500},
author={Jae-Yong Baek and Du-Hwan Hur and Deok-Woong Kim and Yong-Sang Yoo and Hyuk-Jin Shin and Dae-Hyeon Park and Seung-Hwan Bae},
title={Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization},
journal={Journal of The Korea Society of Computer and Information},
issn={1598-849X},
year={2024},
volume={29},
number={7},
pages={11-20},
doi={10.9708/jksci.2024.29.07.011}
TY - JOUR
AU - Jae-Yong Baek
AU - Du-Hwan Hur
AU - Deok-Woong Kim
AU - Yong-Sang Yoo
AU - Hyuk-Jin Shin
AU - Dae-Hyeon Park
AU - Seung-Hwan Bae
TI - Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization
JO - Journal of The Korea Society of Computer and Information
PY - 2024
VL - 29
IS - 7
PB - The Korean Society Of Computer And Information
SP - 11
EP - 20
SN - 1598-849X
AB - In this paper, we propose the BAG (Bit-width Aware Generator) and the Intermediate Layer Knowledge Distillation using Channel-wise Attention to reduce the knowledge gap between a quantized network, a full-precision network, and a generator in GDFQ (Generative Data-Free Quantization). Since the generator in GDFQ is only trained by the feedback from the full-precision network, the gap resulting in decreased capability due to low bit-width of the quantized network has no effect on training the generator. To alleviate this problem, BAG is quantized with same bit-width of the quantized network, and it can generate synthetic images, which are effectively used for training the quantized network. Typically, the knowledge gap between the quantized network and the full-precision network is also important. To resolve this, we compute channel-wise attention of outputs of convolutional layers, and minimize the loss function as the distance of them. As the result, the quantized network can learn which channels to focus on more from mimicking the full-precision network. To prove the efficiency of proposed methods, we quantize the network trained on CIFAR-100 with 3 bit-width weights and activations, and train it and the generator with our method. As the result, we achieve 56.14% Top-1 Accuracy and increase 3.4% higher accuracy compared to our baseline AdaDFQ.
KW - Neural Network Quantization;Data-free Quantization;Generative model;Knowledge Distillation;Attention Mechanism
DO - 10.9708/jksci.2024.29.07.011
ER -
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park and Seung-Hwan Bae. (2024). Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization. Journal of The Korea Society of Computer and Information, 29(7), 11-20.
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park and Seung-Hwan Bae. 2024, "Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization", Journal of The Korea Society of Computer and Information, vol.29, no.7 pp.11-20. Available from: doi:10.9708/jksci.2024.29.07.011
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park, Seung-Hwan Bae "Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization" Journal of The Korea Society of Computer and Information 29.7 pp.11-20 (2024) : 11.
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park, Seung-Hwan Bae. Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization. 2024; 29(7), 11-20. Available from: doi:10.9708/jksci.2024.29.07.011
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park and Seung-Hwan Bae. "Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization" Journal of The Korea Society of Computer and Information 29, no.7 (2024) : 11-20.doi: 10.9708/jksci.2024.29.07.011
Jae-Yong Baek; Du-Hwan Hur; Deok-Woong Kim; Yong-Sang Yoo; Hyuk-Jin Shin; Dae-Hyeon Park; Seung-Hwan Bae. Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization. Journal of The Korea Society of Computer and Information, 29(7), 11-20. doi: 10.9708/jksci.2024.29.07.011
Jae-Yong Baek; Du-Hwan Hur; Deok-Woong Kim; Yong-Sang Yoo; Hyuk-Jin Shin; Dae-Hyeon Park; Seung-Hwan Bae. Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization. Journal of The Korea Society of Computer and Information. 2024; 29(7) 11-20. doi: 10.9708/jksci.2024.29.07.011
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park, Seung-Hwan Bae. Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization. 2024; 29(7), 11-20. Available from: doi:10.9708/jksci.2024.29.07.011
Jae-Yong Baek, Du-Hwan Hur, Deok-Woong Kim, Yong-Sang Yoo, Hyuk-Jin Shin, Dae-Hyeon Park and Seung-Hwan Bae. "Bit-width Aware Generator and Intermediate Layer Knowledge Distillation using Channel-wise Attention for Generative Data-Free Quantization" Journal of The Korea Society of Computer and Information 29, no.7 (2024) : 11-20.doi: 10.9708/jksci.2024.29.07.011
