mumble/AudioStats.cpp

/* Copyright (C) 2005-2006, Thorvald Natvig <thorvald@natvig.com>

   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions
   are met:

   - Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
   - Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.
   - Neither the name of the Mumble Developers nor the names of its
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "AudioStats.h"
#include "AudioInput.h"
#include "Global.h"

/** Speex echo cancellation state; sync from mdf.c in Speex distribution.
  */

typedef float spx_word16_t;
typedef float spx_word32_t;
typedef float spx_float_t;
typedef spx_word32_t spx_mem_t;
/** Speex echo cancellation state. */
struct CloneSpeexEchoState {
   int frame_size;           /**< Number of samples processed each time */
   int window_size;
   int M;
   int cancel_count;
   int adapted;
   int saturated;
   int screwed_up;
   spx_int32_t sampling_rate;
   spx_word16_t spec_average;
   spx_word16_t beta0;
   spx_word16_t beta_max;
   spx_word32_t sum_adapt;
   spx_word16_t leak_estimate;

   spx_word16_t *e;      /* scratch */
   spx_word16_t *x;      /* Far-end input buffer (2N) */
   spx_word16_t *X;      /* Far-end buffer (M+1 frames) in frequency domain */
   spx_word16_t *input;  /* scratch */
   spx_word16_t *y;      /* scratch */
   spx_word16_t *last_y;
   spx_word16_t *Y;      /* scratch */
   spx_word16_t *E;
   spx_word32_t *PHI;    /* scratch */
   spx_word32_t *W;      /* (Background) filter weights */
   spx_word16_t *foreground; /* Foreground filter weights */
   spx_word32_t  Davg1;  /* 1st recursive average of the residual power difference */
   spx_word32_t  Davg2;  /* 2nd recursive average of the residual power difference */
   spx_float_t   Dvar1;  /* Estimated variance of 1st estimator */
   spx_float_t   Dvar2;  /* Estimated variance of 2nd estimator */
   spx_word32_t *power;  /* Power of the far-end signal */
   spx_float_t  *power_1;/* Inverse power of far-end */
   spx_word16_t *wtmp;   /* scratch */
   spx_word32_t *Rf;     /* scratch */
   spx_word32_t *Yf;     /* scratch */
   spx_word32_t *Xf;     /* scratch */
   spx_word32_t *Eh;
   spx_word32_t *Yh;
   spx_float_t   Pey;
   spx_float_t   Pyy;
   spx_word16_t *window;
   spx_word16_t *prop;
   void *fft_table;
   spx_word16_t memX, memD, memE;
   spx_word16_t preemph;
   spx_word16_t notch_radius;
   spx_mem_t notch_mem[2];
};

/** Speex pre-processor state. */
struct CloneSpeexPreprocessState {
   /* Basic info */
   int    frame_size;        /**< Number of samples processed each time */
   int    ps_size;           /**< Number of points in the power spectrum */
   int    sampling_rate;     /**< Sampling rate of the input/output */
   int    nbands;
   void *bank;
   int    denoise_enabled;
   int    vad_enabled;
   int    dereverb_enabled;
   spx_word16_t  reverb_decay;
   spx_word16_t  reverb_level;
   spx_word16_t speech_prob_start;
   spx_word16_t speech_prob_continue;
   int    noise_suppress;
   int    echo_suppress;
   int    echo_suppress_active;
   CloneSpeexEchoState *echo_state;
   spx_word16_t *frame;      /**< Processing frame (2*ps_size) */
   spx_word16_t *ft;         /**< Processing frame in freq domain (2*ps_size) */
   spx_word32_t *ps;         /**< Current power spectrum */
   spx_word16_t *gain2;      /**< Adjusted gains */
   spx_word16_t *gain_floor; /**< Minimum gain allowed */
   spx_word16_t *window;     /**< Analysis/Synthesis window */
   spx_word32_t *noise;      /**< Noise estimate */
   spx_word32_t *reverb_estimate; /**< Estimate of reverb energy */
   spx_word32_t *old_ps;     /**< Power spectrum for last frame */
   spx_word16_t *gain;       /**< Ephraim Malah gain */
   spx_word16_t *prior;      /**< A-priori SNR */
   spx_word16_t *post;       /**< A-posteriori SNR */

   spx_word32_t *S;          /**< Smoothed power spectrum */
   spx_word32_t *Smin;       /**< See Cohen paper */
   spx_word32_t *Stmp;       /**< See Cohen paper */
   int *update_prob;       /**< Propability of speech presence for noise update */

   spx_word16_t *zeta;       /**< Smoothed a priori SNR */
   spx_word32_t *echo_noise;
   spx_word32_t *residual_echo;

   spx_word16_t *inbuf;      /**< Input buffer (overlapped analysis) */
   spx_word16_t *outbuf;     /**< Output buffer (for overlap and add) */

   int    agc_enabled;
   float  agc_level;
   float  loudness_accum;
   float *loudness_weight;   /**< Perceptual loudness curve */
   float  loudness;          /**< Loudness estimate */
   float  agc_gain;          /**< Current AGC gain */
   int    nb_loudness_adapt; /**< Number of frames used for loudness adaptation so far */
   float  max_gain;          /**< Maximum gain allowed */
   float  max_increase_step; /**< Maximum increase in gain from one frame to another */
   float  max_decrease_step; /**< Maximum decrease in gain from one frame to another */
   float  prev_loudness;     /**< Loudness of previous frame */
   float  init_max;          /**< Current gain limit during initialisation */
   int    nb_adapt;          /**< Number of frames used for adaptation so far */
   int    was_speech;
   int    min_count;         /**< Number of frames processed so far */
   void  *fft_lookup;        /**< Lookup table for the FFT */
};

AudioEchoWidget::AudioEchoWidget(QWidget *p) : QGLWidget(p) {
	setMinimumSize(100, 60);
	mode = MODULUS;
}

void AudioEchoWidget::initializeGL() {
    glDisable(GL_LIGHTING);

	glClearColor(0,0,0,0);
    glShadeModel(GL_SMOOTH);

    glEnable(GL_COLOR_MATERIAL);
    glDisable(GL_CULL_FACE);
}

void AudioEchoWidget::resizeGL(int w, int h) {
    glViewport(0, 0, w, h);
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    glOrtho(0.0, 1.0, 0.0, 1.0, -1.0, 1.0);
    glMatrixMode(GL_MODELVIEW);
	glLoadIdentity();
}

static inline void mapEchoToColor(float echo) {
	bool neg = (echo < 0);
	echo = fabs(echo);

	float a, b, c;

	if (echo > 1.0) {
		echo = 1.0;
		c = 0.5;
	} else {
		c = 0.0;
	}

	if (echo < 0.5) {
		a = echo * 2;
		b = 0;
	} else {
		a = 1;
		b = (echo - 0.5) * 2;
	}

	if (neg)
		glColor3f(a, b, c);
	else
		glColor3f(c, b, a);
}

#define WGT(x,y) st->W[(y)*N + 2*(x)+1]

void AudioEchoWidget::paintGL() {
	AudioInputPtr ai = g.ai;
	if (! ai || ! ai->sesEcho)
		return;

	CloneSpeexEchoState *st = reinterpret_cast<CloneSpeexEchoState *>(ai->sesEcho);

    int N = st->window_size;
    int n = N / 2;
    int M = st->M;

    double xscale = 1.0 / n;
    double yscale = 1.0 / M;


    glBegin(GL_QUADS);

    for(int j = 0; j < M; j++) {
    	for(int i=0;i < n; i++) {
			double xa = i * xscale;
			double ya = j * yscale;

			double xb = xa + xscale;
			double yb = ya + yscale;

			float real = 1.0;
			float imag = 0.0;
			if (i == 0)
				real = st->W[j*N];
			else if (i == n-1)
				real = st->W[j*N + 2*i];
			else {
				real = st->W[j*N + 2*i - 1];
				imag = st->W[j*N + 2*i];
			}

			float v;

			switch (mode) {
				case REAL:
					v = real;
					break;
				case IMAGINARY:
					v = imag;
					break;
				case MODULUS:
					v = sqrt(real*real+imag*imag);
					break;
				case PHASE:
					v = atan2(imag,real)/M_PI;
					break;
			}

			mapEchoToColor(v);
			glVertex2f(xa, ya);
			glVertex2f(xb, ya);
			glVertex2f(xb, yb);
			glVertex2f(xa, yb);
		}
	}

	glEnd();
}

AudioNoiseWidget::AudioNoiseWidget(QWidget *p) : QWidget(p) {
	setMinimumSize(100,60);
}

void AudioNoiseWidget::paintEvent(QPaintEvent *evt) {
	QPainter paint(this);
	QPalette pal;

	paint.fillRect(rect(), pal.color(QPalette::Background));

	AudioInputPtr ai = g.ai;
	if (ai.get() == NULL || ! ai->isRunning() || ! ai->sppPreprocess)
		return;

	QPolygonF poly;

	CloneSpeexPreprocessState *st=reinterpret_cast<CloneSpeexPreprocessState *>(ai->sppPreprocess);

	qreal sx, sy;

	sx = (width() - 1.0f) / (st->ps_size * 1.0f);
	sy = height() - 1;

	poly << QPointF(0.0f, height() - 1);
#ifdef SPEEX_ANCIENT_PP
	float fftmul = 1.0 / (st->ps_size * 32768.0);
#else
	float fftmul = 1.0 / (32768.0);
#endif
	for(int i=0; i < st->ps_size; i++) {
		qreal xp, yp;
		xp = i * sx;
		yp = sqrt(st->noise[i]) - 1;
		yp = yp * fftmul;
		yp = fmin(yp * 30.0, 1.0);
		yp = (1 - yp) * sy;
		poly << QPointF(xp, yp);
	}

	poly << QPointF(width() - 1, height() - 1);
	poly << QPointF(0.0f, height() - 1);

	paint.setPen(Qt::blue);
	paint.setBrush(Qt::blue);
	paint.drawPolygon(poly);

	poly.clear();

	for(int i=0;i < st->ps_size; i++) {
		qreal xp, yp;
		xp = i * sx;
		yp = sqrt(st->ps[i]) - 1;
		yp = yp * fftmul;
		yp = fmin(yp * 30.0, 1.0);
		yp = (1 - yp) * sy;
		poly << QPointF(xp, yp);
	}

	paint.setPen(Qt::red);
	paint.drawPolyline(poly);
}

AudioStats::AudioStats(QWidget *p) : QDialog(p) {
	setAttribute(Qt::WA_DeleteOnClose, true);

	setWindowTitle("Mumble");

    QLabel *lab;
	QGridLayout *l=new QGridLayout;
    setLayout(l);

	lab = new QLabel(tr("Peak microphone level"), this);
	l->addWidget(lab, 0, 0);
	qlMicLevel = new QLabel(this);
	l->addWidget(qlMicLevel, 0, 1);

	lab = new QLabel(tr("Peak speaker level"), this);
	l->addWidget(lab, 1, 0);
	qlSpeakerLevel = new QLabel(this);
	l->addWidget(qlSpeakerLevel, 1, 1);

	lab = new QLabel(tr("Peak clean level"), this);
	l->addWidget(lab, 2, 0);
	qlSignalLevel = new QLabel(this);
	l->addWidget(qlSignalLevel, 2, 1);

	lab = new QLabel(tr("Microphone loudness"), this);
	l->addWidget(lab, 3, 0);
	qlMicVolume = new QLabel(this);
	l->addWidget(qlMicVolume, 3, 1);

	lab = new QLabel(tr("Mic Signal-To-Noise"), this);
	l->addWidget(lab, 4, 0);
	qlMicSNR = new QLabel(this);
	l->addWidget(qlMicSNR, 4, 1);

	lab = new QLabel(tr("Speech Probability"), this);
	l->addWidget(lab, 5, 0);
	qlSpeechProb = new QLabel(this);
	l->addWidget(qlSpeechProb, 5, 1);

	lab = new QLabel(tr("Audio bitrate"), this);
	l->addWidget(lab, 6, 0);
	qlBitrate = new QLabel(this);
	l->addWidget(qlBitrate, 6, 1);

	anwNoise = new AudioNoiseWidget(this);
	l->addWidget(anwNoise,7,0,1,2);
	l->setRowStretch(7, 1);

	AudioInputPtr ai = g.ai;
	if (ai && ai->sesEcho) {

		QHBoxLayout *hbox = new QHBoxLayout;

		QRadioButton *b;

		b = new QRadioButton("Real");
		qmEchoMode[b] = AudioEchoWidget::REAL;
		connect(b, SIGNAL(clicked(bool)), this, SLOT(onEchoMode(bool)));
		hbox->addWidget(b);

		b = new QRadioButton("Imaginary");
		qmEchoMode[b] = AudioEchoWidget::IMAGINARY;
		connect(b, SIGNAL(clicked(bool)), this, SLOT(onEchoMode(bool)));
		hbox->addWidget(b);

		b = new QRadioButton("Modulus");
		b->setChecked(true);
		qmEchoMode[b] = AudioEchoWidget::MODULUS;
		connect(b, SIGNAL(clicked(bool)), this, SLOT(onEchoMode(bool)));
		hbox->addWidget(b);

		b = new QRadioButton("Phase");
		qmEchoMode[b] = AudioEchoWidget::PHASE;
		connect(b, SIGNAL(clicked(bool)), this, SLOT(onEchoMode(bool)));
		hbox->addWidget(b);

		l->addLayout(hbox, 8, 0, 1, 2);

		aewEcho = new AudioEchoWidget(this);
		l->addWidget(aewEcho,9,0,1,2);
		l->setRowStretch(9, 1);
	} else {
		aewEcho = NULL;
	}

	qtTick = new QTimer(this);
	qtTick->setObjectName("Tick");
	qtTick->start(50);

	qlMicLevel->setToolTip(tr("Peak power in last frame"));
	qlMicLevel->setWhatsThis(tr("This shows the peak power in the last frame (20 ms), and is the same measurement "
								"as you would usually find displayed as \"input power\". Please disregard this and "
								"look at <b>Loudness</b> instead, which is much more steady and disregards outliers."
								));
	qlSpeakerLevel->setToolTip(tr("Peak power in last frame"));
	qlSpeakerLevel->setWhatsThis(tr("This shows the peak power in the last frame (20 ms) of the speakers. Unless you "
								"are using a multi-channel sampling method (such as ASIO) with speaker channels "
								"configured, this will be 0. If you have such a setup configured, and this still "
								"shows 0 while you're playing audio from other programs, your setup is not working."));
	qlSpeakerLevel->setToolTip(tr("Peak power in last frame"));
	qlSpeakerLevel->setWhatsThis(tr("This shows the peak power in the last frame (20 ms) after all processing. Ideally, "
								"this should be -96 dB when you're not talking. In reality, a sound studio should see "
								"-60 dB, and you should hopefully see somewhere around -20 dB. When you are talking, this "
								"should rise to somewhere between -5 and -10 dB.<br />"
								"If you are using echo cancellation, and this rises to more than -15 dB when you're not "
								"talking, your setup is not working, and you'll annoy other players with echoes."));
	qlMicVolume->setToolTip(tr("How close the current input level is to ideal"));
	qlMicVolume->setWhatsThis(tr("This shows how close your current input volume is to the ideal. To adjust your "
								 "microphone level, open whatever program you use to adjust the recording volume, "
								 "and look at the value here while talking.<br />"
								 "<b>Talk loud, as you would when you're upset over getting fragged by a noob.</b><br />"
								 "Adjust the volume until this value is close to 100%, but make sure it doesn't go above. "
								 "If it does go above, you are likely to get clipping in parts of your speech, which will "
								 "degrade sound quality"));
	qlMicSNR->setToolTip(tr("Signal-To-Noise ratio from the microphone"));
	qlMicSNR->setWhatsThis(tr("This is the Signal-To-Noise Ratio (SNR) of the microphone in the last frame (20 ms). "
								"It shows how much clearer "
								"the voice is compared to the noise.<br />If this value is below 1.0, there's more noise "
								"than voice in the signal, and so quality is reduced.<br />There is no upper limit to this "
								"value, but don't expect to see much above 40-50 without a sound studio."));
	qlSpeechProb->setToolTip(tr("Probability of speech"));
	qlSpeechProb->setWhatsThis(tr("This is the probability that the last frame (20 ms) was speech and not environment noise.<br />"
								  "Voice activity transmission depends on this being right. The trick with this is that the middle "
								  "of a sentence is always detected as speech; the problem is the pauses between words and the "
								  "start of speech. It's hard to distinguish a sigh from a word starting with 'h'.<br />"
								  "If this is in bold font, it means Mumble is currently transmitting (if you're connected)."));
	qlBitrate->setToolTip(tr("Bitrate of last frame"));
	qlBitrate->setWhatsThis(tr("This is the audio bitrate of the last compressed frame (20 ms), and as such will jump up and down "
							   "as the VBR adjusts the quality. To adjust the peak bitrate, adjust <b>Compression Complexity</b> "
							   "in the Settings dialog."));

	anwNoise->setToolTip(tr("Power spectrum of input signal and noise estimate"));
	anwNoise->setWhatsThis(tr("This shows the power spectrum of the current input signal (red line) and the current noise estimate "
							  "(filled blue).<br />"
							  "All amplitudes are multiplied by 30 to show the interesting parts (how much more signal than noise "
							  "is present in each waveband).<br />"
							  "This is probably only of interest if you're trying to fine-tune noise conditions on your microphone. "
							  "Under good conditions, there should be just a tiny flutter of blue at the bottom. If the blue is more than "
							  "halfway up on the graph, you have a seriously noisy environment."));

	if (aewEcho) {
		aewEcho->setToolTip(tr("Weights of the echo canceller"));
		aewEcho->setWhatsThis(tr("This shows the weights of the echo canceller, with time increasing downwards and frequency increasing to the right.<br />"
								  "Ideally, this should be black, indicating no echo exists at all. More commonly, you'll have one or more horizontal stripes "
								  "of bluish color representing time delayed echo. You should be able to see the weights updated in real time.<br />"
								  "Please note that as long as you have nothing to echo off, you won't see much useful data here. Play some music and "
								  "things should stabilize. <br />"
								  "You can choose to view the real or imaginary parts of the frequency-domain weights, or alternately the computed modulus and "
								  "phase. The most useful of these will likely be modulus, which is the amplitude of the echo, and shows you how much of the "
								  "outgoing signal is being removed at that time step. The other viewing modes are mostly useful to people who want to tune the "
								  "echo cancellation algorithms.<br />"
								  "Please note: If the entire image fluctuates massively while in modulus mode, "
								  "the echo canceller fails to find any correlation whatsoever between the two input "
								  "sources (speakers and microphone). Either you have a very long delay on the echo, or one of the input sources is configured wrong."
								  ));
	}

    QMetaObject::connectSlotsByName(this);

	bTalking = false;

    on_Tick_timeout();
}

void AudioStats::on_Tick_timeout() {
	AudioInputPtr ai = g.ai;

	if (ai.get() == NULL || ! ai->isRunning() || ! ai->sppPreprocess)
		return;

	CloneSpeexPreprocessState *st=reinterpret_cast<CloneSpeexPreprocessState *>(ai->sppPreprocess);
	bool nTalking = ai->isTransmitting();

	QString txt;

	txt.sprintf("%06.2f dB",ai->dPeakMic);
	qlMicLevel->setText(txt);

	txt.sprintf("%06.2f dB",ai->dPeakSpeaker);
	qlSpeakerLevel->setText(txt);

	txt.sprintf("%06.2f dB",ai->dPeakSignal);
	qlSignalLevel->setText(txt);

	float Zframe = 0;
	int freq_start = static_cast<int>(300.0f*2.f*st->ps_size/st->sampling_rate);
	int freq_end   = static_cast<int>(2000.0f*2.f*st->ps_size/st->sampling_rate);
	for (int i=freq_start;i<freq_end;i++)
		Zframe += st->zeta[i];
	Zframe /= (freq_end-freq_start);
	txt.sprintf("%06.3f",Zframe);
	qlMicSNR->setText(txt);

	txt.sprintf("%03.0f%%",nTalking ? 100.0 : 0.0);
	qlSpeechProb->setText(txt);

	txt.sprintf("%04.1f kbit/s",ai->iBitrate / 1000.0);
	qlBitrate->setText(txt);

	if (nTalking != bTalking) {
		bTalking = nTalking;
		QFont f = qlSpeechProb->font();
		f.setBold(bTalking);
		qlSpeechProb->setFont(f);
	}

	anwNoise->update();
	if (aewEcho)
		aewEcho->updateGL();
}

void AudioStats::onEchoMode(bool) {
	if (aewEcho)
		aewEcho->mode = qmEchoMode[sender()];
}