2018
Koch, Tobias; Vazquez-Vilar, Gonzalo
A Rigorous Approach to High-Resolution Entropy-Constrained Vector Quantization Artículo de revista
En: IEEE Transactions on Information Theory, vol. 64, no 4, pp. 2609-2625, 2018, ISSN: 0018-9448.
Enlaces | BibTeX | Etiquetas: Distortion, Distortion measurement, Entropy, Entropy constrained, high resolution, Probability density function, quantization, Rate-distortion, Rate-distortion theory, Vector quantization
@article{koch-TIT2018a,
title = {A Rigorous Approach to High-Resolution Entropy-Constrained Vector Quantization},
author = {Tobias Koch and Gonzalo Vazquez-Vilar},
doi = {10.1109/TIT.2018.2803064},
issn = {0018-9448},
year = {2018},
date = {2018-04-01},
journal = {IEEE Transactions on Information Theory},
volume = {64},
number = {4},
pages = {2609-2625},
keywords = {Distortion, Distortion measurement, Entropy, Entropy constrained, high resolution, Probability density function, quantization, Rate-distortion, Rate-distortion theory, Vector quantization},
pubstate = {published},
tppubtype = {article}
}
2014
Alvarado, Alex; Brannstrom, Fredrik; Agrell, Erik; Koch, Tobias
High-SNR Asymptotics of Mutual Information for Discrete Constellations With Applications to BICM Artículo de revista
En: IEEE Transactions on Information Theory, vol. 60, no 2, pp. 1061–1076, 2014, ISSN: 0018-9448.
Resumen | Enlaces | BibTeX | Etiquetas: additive white Gaussian noise channel, Anti-Gray code, bit-interleaved coded modulation, discrete constellations, Entropy, Gray code, high-SNR asymptotics, IP networks, Labeling, minimum-mean square error, Modulation, Mutual information, Signal to noise ratio, Vectors
@article{Alvarado2014,
title = {High-SNR Asymptotics of Mutual Information for Discrete Constellations With Applications to BICM},
author = {Alex Alvarado and Fredrik Brannstrom and Erik Agrell and Tobias Koch},
url = {http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=6671479
http://www.tsc.uc3m.es/~koch/files/IEEE_TIT_60%282%29.pdf},
issn = {0018-9448},
year = {2014},
date = {2014-01-01},
journal = {IEEE Transactions on Information Theory},
volume = {60},
number = {2},
pages = {1061--1076},
abstract = {Asymptotic expressions of the mutual information between any discrete input and the corresponding output of the scalar additive white Gaussian noise channel are presented in the limit as the signal-to-noise ratio (SNR) tends to infinity. Asymptotic expressions of the symbol-error probability (SEP) and the minimum mean-square error (MMSE) achieved by estimating the channel input given the channel output are also developed. It is shown that for any input distribution, the conditional entropy of the channel input given the output, MMSE, and SEP have an asymptotic behavior proportional to the Gaussian Q-function. The argument of the Q-function depends only on the minimum Euclidean distance (MED) of the constellation and the SNR, and the proportionality constants are functions of the MED and the probabilities of the pairs of constellation points at MED. The developed expressions are then generalized to study the high-SNR behavior of the generalized mutual information (GMI) for bit-interleaved coded modulation (BICM). By means of these asymptotic expressions, the long-standing conjecture that Gray codes are the binary labelings that maximize the BICM-GMI at high SNR is proven. It is further shown that for any equally spaced constellation whose size is a power of two, there always exists an anti-Gray code giving the lowest BICM-GMI at high SNR.},
keywords = {additive white Gaussian noise channel, Anti-Gray code, bit-interleaved coded modulation, discrete constellations, Entropy, Gray code, high-SNR asymptotics, IP networks, Labeling, minimum-mean square error, Modulation, Mutual information, Signal to noise ratio, Vectors},
pubstate = {published},
tppubtype = {article}
}
Asymptotic expressions of the mutual information between any discrete input and the corresponding output of the scalar additive white Gaussian noise channel are presented in the limit as the signal-to-noise ratio (SNR) tends to infinity. Asymptotic expressions of the symbol-error probability (SEP) and the minimum mean-square error (MMSE) achieved by estimating the channel input given the channel output are also developed. It is shown that for any input distribution, the conditional entropy of the channel input given the output, MMSE, and SEP have an asymptotic behavior proportional to the Gaussian Q-function. The argument of the Q-function depends only on the minimum Euclidean distance (MED) of the constellation and the SNR, and the proportionality constants are functions of the MED and the probabilities of the pairs of constellation points at MED. The developed expressions are then generalized to study the high-SNR behavior of the generalized mutual information (GMI) for bit-interleaved coded modulation (BICM). By means of these asymptotic expressions, the long-standing conjecture that Gray codes are the binary labelings that maximize the BICM-GMI at high SNR is proven. It is further shown that for any equally spaced constellation whose size is a power of two, there always exists an anti-Gray code giving the lowest BICM-GMI at high SNR.
A, Pastore; Koch, Tobias; Fonollosa, Javier Rodriguez
A Rate-Splitting Approach to Fading Channels With Imperfect Channel-State Information Artículo de revista
En: IEEE Transactions on Information Theory, vol. 60, no 7, pp. 4266–4285, 2014, ISSN: 0018-9448.
Resumen | Enlaces | BibTeX | Etiquetas: channel capacity, COMONSENS, DEIPRO, Entropy, Fading, fading channels, flat fading, imperfect channel-state information, MobileNET, Mutual information, OTOSiS, Random variables, Receivers, Signal to noise ratio, Upper bound
@article{Pastore2014a,
title = {A Rate-Splitting Approach to Fading Channels With Imperfect Channel-State Information},
author = {Pastore A and Tobias Koch and Javier Rodriguez Fonollosa},
url = {http://ieeexplore.ieee.org/articleDetails.jsp?arnumber=6832779 http://www.tsc.uc3m.es/~koch/files/IEEE_TIT_60(7).pdf http://arxiv.org/pdf/1301.6120.pdf},
issn = {0018-9448},
year = {2014},
date = {2014-01-01},
journal = {IEEE Transactions on Information Theory},
volume = {60},
number = {7},
pages = {4266--4285},
publisher = {IEEE},
abstract = {As shown by M\'{e}dard, the capacity of fading channels with imperfect channel-state information can be lower-bounded by assuming a Gaussian channel input (X) with power (P) and by upper-bounding the conditional entropy (h(X|Y,hat {H})) by the entropy of a Gaussian random variable with variance equal to the linear minimum mean-square error in estimating (X) from ((Y,hat {H})) . We demonstrate that, using a rate-splitting approach, this lower bound can be sharpened: by expressing the Gaussian input (X) as the sum of two independent Gaussian variables (X_1) and (X_2) and by applying M\'{e}dard's lower bound first to bound the mutual information between (X_1) and (Y) while treating (X_2) as noise, and by applying it a second time to the mutual information between (X_2) and (Y) while assuming (X_1) to be known, we obtain a capacity lower bound that is strictly larger than M\'{e}dard's lower bound. We then generalize this approach to an arbi- rary number (L) of layers, where (X) is expressed as the sum of (L) independent Gaussian random variables of respective variances (P_ell ) , (ell = 1,dotsc ,L) summing up to (P) . Among all such rate-splitting bounds, we determine the supremum over power allocations (P_ell ) and total number of layers (L) . This supremum is achieved for (L rightarrow infty ) and gives rise to an analytically expressible capacity lower bound. For Gaussian fading, this novel bound is shown to converge to the Gaussian-input mutual information as the signal-to-noise ratio (SNR) grows, provided that the variance of the channel estimation error (H-hat {H}) tends to zero as the SNR tends to infinity.},
keywords = {channel capacity, COMONSENS, DEIPRO, Entropy, Fading, fading channels, flat fading, imperfect channel-state information, MobileNET, Mutual information, OTOSiS, Random variables, Receivers, Signal to noise ratio, Upper bound},
pubstate = {published},
tppubtype = {article}
}
As shown by Médard, the capacity of fading channels with imperfect channel-state information can be lower-bounded by assuming a Gaussian channel input (X) with power (P) and by upper-bounding the conditional entropy (h(X|Y,hat Ħ)) by the entropy of a Gaussian random variable with variance equal to the linear minimum mean-square error in estimating (X) from ((Y,hat Ħ)) . We demonstrate that, using a rate-splitting approach, this lower bound can be sharpened: by expressing the Gaussian input (X) as the sum of two independent Gaussian variables (X_1) and (X_2) and by applying Médard's lower bound first to bound the mutual information between (X_1) and (Y) while treating (X_2) as noise, and by applying it a second time to the mutual information between (X_2) and (Y) while assuming (X_1) to be known, we obtain a capacity lower bound that is strictly larger than Médard's lower bound. We then generalize this approach to an arbi- rary number (L) of layers, where (X) is expressed as the sum of (L) independent Gaussian random variables of respective variances (P_ell ) , (ell = 1,dotsc ,L) summing up to (P) . Among all such rate-splitting bounds, we determine the supremum over power allocations (P_ell ) and total number of layers (L) . This supremum is achieved for (L rightarrow infty ) and gives rise to an analytically expressible capacity lower bound. For Gaussian fading, this novel bound is shown to converge to the Gaussian-input mutual information as the signal-to-noise ratio (SNR) grows, provided that the variance of the channel estimation error (H-hat Ħ) tends to zero as the SNR tends to infinity.
2012
Leiva-Murillo, Jose M; Artés-Rodríguez, Antonio
Information-Theoretic Linear Feature Extraction Based on Kernel Density Estimators: A Review Artículo de revista
En: IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews), vol. 42, no 6, pp. 1180–1189, 2012, ISSN: 1094-6977.
Resumen | Enlaces | BibTeX | Etiquetas: Bandwidth, Density, detection theory, Entropy, Estimation, Feature extraction, Feature extraction (FE), information theoretic linear feature extraction, information theory, information-theoretic learning (ITL), Kernel, Kernel density estimation, kernel density estimators, Machine learning
@article{Leiva-Murillo2012a,
title = {Information-Theoretic Linear Feature Extraction Based on Kernel Density Estimators: A Review},
author = {Jose M Leiva-Murillo and Antonio Art\'{e}s-Rodr\'{i}guez},
url = {http://www.tsc.uc3m.es/~antonio/papers/P44_2012_Information Theoretic Linear Feature Extraction Based on Kernel Density Estimators A Review.pdf http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=6185689},
issn = {1094-6977},
year = {2012},
date = {2012-01-01},
journal = {IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews)},
volume = {42},
number = {6},
pages = {1180--1189},
abstract = {In this paper, we provide a unified study of the application of kernel density estimators to supervised linear feature extraction by means of criteria inspired by information and detection theory. We enrich this study by the incorporation of two novel criteria to the study, i.e., the mutual information and the likelihood ratio test, and perform both a theoretical and an experimental comparison between the new methods and other ones previously described in the literature. The impact of the bandwidth selection of the density estimator in the classification performance is discussed. Some theoretical results that bound classification performance as a function or mutual information are also compiled. A set of experiments on different real-world datasets allows us to perform an empirical comparison of the methods, in terms of both accuracy and computational complexity. We show the suitability of these methods to determine the dimension of the subspace that contains the discriminative information.},
keywords = {Bandwidth, Density, detection theory, Entropy, Estimation, Feature extraction, Feature extraction (FE), information theoretic linear feature extraction, information theory, information-theoretic learning (ITL), Kernel, Kernel density estimation, kernel density estimators, Machine learning},
pubstate = {published},
tppubtype = {article}
}
In this paper, we provide a unified study of the application of kernel density estimators to supervised linear feature extraction by means of criteria inspired by information and detection theory. We enrich this study by the incorporation of two novel criteria to the study, i.e., the mutual information and the likelihood ratio test, and perform both a theoretical and an experimental comparison between the new methods and other ones previously described in the literature. The impact of the bandwidth selection of the density estimator in the classification performance is discussed. Some theoretical results that bound classification performance as a function or mutual information are also compiled. A set of experiments on different real-world datasets allows us to perform an empirical comparison of the methods, in terms of both accuracy and computational complexity. We show the suitability of these methods to determine the dimension of the subspace that contains the discriminative information.
2007
Leiva-Murillo, Jose M; Artés-Rodríguez, Antonio
Maximization of Mutual Information for Supervised Linear Feature Extraction Artículo de revista
En: IEEE Transactions on Neural Networks, vol. 18, no 5, pp. 1433–1441, 2007, ISSN: 1045-9227.
Resumen | Enlaces | BibTeX | Etiquetas: Algorithms, Artificial Intelligence, Automated, component-by-component gradient-ascent method, Computer Simulation, Data Mining, Entropy, Feature extraction, gradient methods, gradient-based entropy, Independent component analysis, Information Storage and Retrieval, information theory, Iron, learning (artificial intelligence), Linear discriminant analysis, Linear Models, Mutual information, Optimization methods, Pattern recognition, Reproducibility of Results, Sensitivity and Specificity, supervised linear feature extraction, Vectors
@article{Leiva-Murillo2007,
title = {Maximization of Mutual Information for Supervised Linear Feature Extraction},
author = {Jose M Leiva-Murillo and Antonio Art\'{e}s-Rodr\'{i}guez},
url = {http://ieeexplore.ieee.org/articleDetails.jsp?arnumber=4298118},
issn = {1045-9227},
year = {2007},
date = {2007-01-01},
journal = {IEEE Transactions on Neural Networks},
volume = {18},
number = {5},
pages = {1433--1441},
publisher = {IEEE},
abstract = {In this paper, we present a novel scheme for linear feature extraction in classification. The method is based on the maximization of the mutual information (MI) between the features extracted and the classes. The sum of the MI corresponding to each of the features is taken as an heuristic that approximates the MI of the whole output vector. Then, a component-by-component gradient-ascent method is proposed for the maximization of the MI, similar to the gradient-based entropy optimization used in independent component analysis (ICA). The simulation results show that not only is the method competitive when compared to existing supervised feature extraction methods in all cases studied, but it also remarkably outperform them when the data are characterized by strongly nonlinear boundaries between classes.},
keywords = {Algorithms, Artificial Intelligence, Automated, component-by-component gradient-ascent method, Computer Simulation, Data Mining, Entropy, Feature extraction, gradient methods, gradient-based entropy, Independent component analysis, Information Storage and Retrieval, information theory, Iron, learning (artificial intelligence), Linear discriminant analysis, Linear Models, Mutual information, Optimization methods, Pattern recognition, Reproducibility of Results, Sensitivity and Specificity, supervised linear feature extraction, Vectors},
pubstate = {published},
tppubtype = {article}
}
In this paper, we present a novel scheme for linear feature extraction in classification. The method is based on the maximization of the mutual information (MI) between the features extracted and the classes. The sum of the MI corresponding to each of the features is taken as an heuristic that approximates the MI of the whole output vector. Then, a component-by-component gradient-ascent method is proposed for the maximization of the MI, similar to the gradient-based entropy optimization used in independent component analysis (ICA). The simulation results show that not only is the method competitive when compared to existing supervised feature extraction methods in all cases studied, but it also remarkably outperform them when the data are characterized by strongly nonlinear boundaries between classes.