Merge pull request #1625 from lissyx/update-tf-master

Update tf master
This commit is contained in:
lissyx 2018-10-04 13:46:45 +02:00 committed by GitHub
commit dfdab7cb51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 627 additions and 7 deletions

View File

@ -67,8 +67,6 @@ def create_flags():
tf.app.flags.DEFINE_boolean ('test', True, 'whether to test the network')
tf.app.flags.DEFINE_integer ('epoch', 75, 'target epoch to train - if negative, the absolute number of additional epochs will be trained')
tf.app.flags.DEFINE_boolean ('use_warpctc', False, 'whether to use GPU bound Warp-CTC')
tf.app.flags.DEFINE_float ('dropout_rate', 0.05, 'dropout rate for feedforward layers')
tf.app.flags.DEFINE_float ('dropout_rate2', -1.0, 'dropout rate for layer 2 - defaults to dropout_rate')
tf.app.flags.DEFINE_float ('dropout_rate3', -1.0, 'dropout rate for layer 3 - defaults to dropout_rate')
@ -519,11 +517,8 @@ def calculate_mean_edit_distance_and_loss(model_feeder, tower, dropout, reuse):
# Calculate the logits of the batch using BiRNN
logits, _ = BiRNN(batch_x, batch_seq_len, dropout, reuse)
# Compute the CTC loss using either TensorFlow's `ctc_loss` or Baidu's `warp_ctc_loss`.
if FLAGS.use_warpctc:
total_loss = tf.contrib.warpctc.warp_ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
else:
total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
# Compute the CTC loss using TensorFlow's `ctc_loss`
total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
# Calculate the average loss across the batch
avg_loss = tf.reduce_mean(total_loss)

BIN
doc/audioTranscript.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -0,0 +1,67 @@
import sys
import os
import logging
import argparse
import numpy as np
import wavTranscriber
# Debug helpers
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
def main(args):
parser = argparse.ArgumentParser(description='Transcribe long audio files using webRTC VAD')
parser.add_argument('--aggressive', type=int, choices=range(4), required=True,
help='Determines how aggressive filtering out non-speech is. (Interger between 0-3)')
parser.add_argument('--audio', required=True,
help='Path to the audio file to run (WAV format)')
parser.add_argument('--model', required=True,
help='Path to directory that contains all model files (output_graph, lm, trie and alphabet)')
if len(sys.argv[1:])<6:
parser.print_help()
parser.exit()
args = parser.parse_args()
# Point to a path containing the pre-trained models & resolve ~ if used
dirName = os.path.expanduser(args.model)
title_names = ['Filename', 'Duration(s)', 'Inference Time(s)', 'Model Load Time(s)', 'LM Load Time(s)']
print("\n%-30s %-20s %-20s %-20s %s" % (title_names[0], title_names[1], title_names[2], title_names[3], title_names[4]))
# Resolve all the paths of model files
output_graph, alphabet, lm, trie = wavTranscriber.resolve_models(dirName)
# Load output_graph, alpahbet, lm and trie
model_retval = wavTranscriber.load_model(output_graph, alphabet, lm, trie)
inference_time = 0.0
# Run VAD on the input file
waveFile = args.audio
segments, sample_rate, audio_length = wavTranscriber.vad_segment_generator(waveFile, args.aggressive)
f = open(waveFile.rstrip(".wav") + ".txt", 'w')
logging.debug("Saving Transcript @: %s" % waveFile.rstrip(".wav") + ".txt")
for i, segment in enumerate(segments):
# Run deepspeech on the chunk that just completed VAD
logging.debug("Processing chunk %002d" % (i,))
audio = np.frombuffer(segment, dtype=np.int16)
output = wavTranscriber.stt(model_retval[0], audio, sample_rate)
inference_time += output[1]
logging.debug("Transcript: %s" % output[0])
f.write(output[0] + " ")
# Summary of the files processed
f.close()
# Extract filename from the full file path
filename, ext = os.path.split(os.path.basename(waveFile))
logging.debug("************************************************************************************************************")
logging.debug("%-30s %-20s %-20s %-20s %s" % (title_names[0], title_names[1], title_names[2], title_names[3], title_names[4]))
logging.debug("%-30s %-20.3f %-20.3f %-20.3f %-0.3f" % (filename + ext, audio_length, inference_time, model_retval[1], model_retval[2]))
logging.debug("************************************************************************************************************")
print("%-30s %-20.3f %-20.3f %-20.3f %-0.3f" % (filename + ext, audio_length, inference_time, model_retval[1], model_retval[2]))
if __name__ == '__main__':
main(sys.argv[1:])

View File

@ -0,0 +1,257 @@
import sys
import os
import inspect
import logging
import traceback
import numpy as np
import wavTranscriber
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
# Debug helpers
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
def PrintFrame():
# 0 represents this line
# 1 represents line at caller
callerframerecord = inspect.stack()[1]
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
logging.debug(info.function, info.lineno)
class WorkerSignals(QObject):
'''
Defines the signals available from a running worker thread.
Supported signals are:
finished:
No data
error
'tuple' (ecxtype, value, traceback.format_exc())
result
'object' data returned from processing, anything
progress
'object' indicating the transcribed result
'''
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(object)
progress = pyqtSignal(object)
class Worker(QRunnable):
'''
Worker Thread
Inherits from QRunnable to handle worker thread setup, signals and wrap-up
@param callback:
The funtion callback to run on this worker thread.
Supplied args and kwargs will be passed through the runner.
@type calllback: function
@param args: Arguments to pass to the callback function
@param kwargs: Keywords to pass to the callback function
'''
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store the conctructor arguments (re-used for processing)
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# Add the callback to our kwargs
self.kwargs['progress_callback'] = self.signals.progress
@pyqtSlot()
def run(self):
'''
Initialise the runner function with the passed args, kwargs
'''
# Retrieve args/kwargs here; and fire up the processing using them
try:
transcript = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
# Return the result of the processing
self.signals.result.emit(transcript)
finally:
# Done
self.signals.finished.emit()
class App(QMainWindow):
dirName = ""
def __init__(self):
super().__init__()
self.title = 'Deepspeech Transcriber'
self.left = 10
self.top = 10
self.width = 480
self.height = 320
self.initUI()
def initUI(self):
self.setWindowTitle(self.title)
self.setGeometry(self.left, self.top, self.width, self.height)
layout = QGridLayout()
layout.setSpacing(10)
self.textbox = QLineEdit(self, placeholderText="Wave File, Mono @ 16 kHz, 16bit Little-Endian")
self.modelsBox = QLineEdit(self, placeholderText="Directory path for output_graph, alphabet, lm & trie")
self.textboxTranscript = QPlainTextEdit(self, placeholderText="Transcription")
self.button = QPushButton('Browse', self)
self.button.setToolTip('Select a wav file')
self.modelsButton = QPushButton('Browse', self)
self.modelsButton.setToolTip('Select deepspeech models folder')
self.transcribeButton = QPushButton('Transcribe', self)
self.transcribeButton.setToolTip('Start Transcription')
layout.addWidget(self.textbox, 0, 0)
layout.addWidget(self.button, 0, 1)
layout.addWidget(self.modelsBox, 1, 0)
layout.addWidget(self.modelsButton, 1, 1)
layout.addWidget(self.transcribeButton, 2, 0, Qt.AlignHCenter)
layout.addWidget(self.textboxTranscript, 3, 0, -1, 0)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
# Connect Button to Function on_click
self.button.clicked.connect(self.on_click)
# Connect the Models Button
self.modelsButton.clicked.connect(self.models_on_click)
# Connect Transcription button to threadpool
self.transcribeButton.clicked.connect(self.transcriptionStart_on_click)
self.show()
# Setup Threadpool
self.threadpool = QThreadPool()
logging.debug("Multithreading with maximum %d threads" % self.threadpool.maxThreadCount())
@pyqtSlot()
def on_click(self):
logging.debug('Browse button clicked')
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
self.fileName, _ = QFileDialog.getOpenFileName(self,"Select wav file to be Transcribed", \
"","All Files (*.wav)", options=options)
if self.fileName:
self.textbox.setText(self.fileName)
logging.debug(self.fileName)
@pyqtSlot()
def models_on_click(self):
logging.debug('Models Browse Button clicked')
self.dirName = QFileDialog.getExistingDirectory(self,"Select deepspeech models directory")
if self.dirName:
self.modelsBox.setText(self.dirName)
logging.debug(self.dirName)
@pyqtSlot()
def transcriptionStart_on_click(self):
logging.debug('Transcription Start button clicked')
# Clear out older data
self.textboxTranscript.setPlainText("")
self.show()
# Threaded signal passing worker functions
worker = Worker(self.runDeepspeech, self.fileName)
worker.signals.result.connect(self.transcription)
worker.signals.finished.connect(self.threadComplete)
worker.signals.progress.connect(self.progress)
# Execute
self.threadpool.start(worker)
def transcription(self, out):
logging.debug("Transcribed text: %s" % out)
self.textboxTranscript.insertPlainText(out)
self.show()
def threadComplete(self):
logging.debug("File processed")
def progress(self, chunk):
logging.debug("Progress: %s" % chunk)
self.textboxTranscript.insertPlainText(chunk)
self.show()
def runDeepspeech(self, waveFile, progress_callback):
# Deepspeech will be run from this method
logging.debug("Preparing for transcription...")
# Go and fetch the models from the directory specified
if self.dirName:
# Resolve all the paths of model files
output_graph, alphabet, lm, trie = wavTranscriber.resolve_models(self.dirName)
else:
logging.critical("*****************************************************")
logging.critical("Model path not specified..")
logging.critical("You sure of what you're doing ?? ")
logging.critical("Trying to fetch from present working directory.")
logging.critical("*****************************************************")
return "Transcription Failed, models path not specified"
# Load output_graph, alpahbet, lm and trie
model_retval = wavTranscriber.load_model(output_graph, alphabet, lm, trie)
inference_time = 0.0
# Run VAD on the input file
segments, sample_rate, audio_length = wavTranscriber.vad_segment_generator(waveFile, 1)
f = open(waveFile.rstrip(".wav") + ".txt", 'w')
logging.debug("Saving Transcript @: %s" % waveFile.rstrip(".wav") + ".txt")
for i, segment in enumerate(segments):
# Run deepspeech on the chunk that just completed VAD
logging.debug("Processing chunk %002d" % (i,))
audio = np.frombuffer(segment, dtype=np.int16)
output = wavTranscriber.stt(model_retval[0], audio, sample_rate)
inference_time += output[1]
f.write(output[0] + " ")
progress_callback.emit(output[0] + " ")
# Summary of the files processed
f.close()
# Format pretty, extract filename from the full file path
filename, ext = os.path.split(os.path.basename(waveFile))
title_names = ['Filename', 'Duration(s)', 'Inference Time(s)', 'Model Load Time(s)', 'LM Load Time(s)']
logging.debug("************************************************************************************************************")
logging.debug("%-30s %-20s %-20s %-20s %s" % (title_names[0], title_names[1], title_names[2], title_names[3], title_names[4]))
logging.debug("%-30s %-20.3f %-20.3f %-20.3f %-0.3f" % (filename + ext, audio_length, inference_time, model_retval[1], model_retval[2]))
logging.debug("************************************************************************************************************")
print("\n%-30s %-20s %-20s %-20s %s" % (title_names[0], title_names[1], title_names[2], title_names[3], title_names[4]))
print("%-30s %-20.3f %-20.3f %-20.3f %-0.3f" % (filename + ext, audio_length, inference_time, model_retval[1], model_retval[2]))
return "\n*********************\nTranscription Done..."
def main(args):
app = QApplication(sys.argv)
w = App()
sys.exit(app.exec_())
if __name__ == '__main__':
main(sys.argv[1:])

View File

@ -0,0 +1,3 @@
deepspeech==0.2.0
webrtcvad
pyqt5

View File

@ -0,0 +1,134 @@
import collections
import contextlib
import wave
def read_wave(path):
"""Reads a .wav file.
Takes the path, and returns (PCM audio data, sample rate).
"""
with contextlib.closing(wave.open(path, 'rb')) as wf:
num_channels = wf.getnchannels()
assert num_channels == 1
sample_width = wf.getsampwidth()
assert sample_width == 2
sample_rate = wf.getframerate()
assert sample_rate in (8000, 16000, 32000)
frames = wf.getnframes()
pcm_data = wf.readframes(frames)
duration = frames / sample_rate
return pcm_data, sample_rate, duration
def write_wave(path, audio, sample_rate):
"""Writes a .wav file.
Takes path, PCM audio data, and sample rate.
"""
with contextlib.closing(wave.open(path, 'wb')) as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(sample_rate)
wf.writeframes(audio)
class Frame(object):
"""Represents a "frame" of audio data."""
def __init__(self, bytes, timestamp, duration):
self.bytes = bytes
self.timestamp = timestamp
self.duration = duration
def frame_generator(frame_duration_ms, audio, sample_rate):
"""Generates audio frames from PCM audio data.
Takes the desired frame duration in milliseconds, the PCM data, and
the sample rate.
Yields Frames of the requested duration.
"""
n = int(sample_rate * (frame_duration_ms / 1000.0) * 2)
offset = 0
timestamp = 0.0
duration = (float(n) / sample_rate) / 2.0
while offset + n < len(audio):
yield Frame(audio[offset:offset + n], timestamp, duration)
timestamp += duration
offset += n
def vad_collector(sample_rate, frame_duration_ms,
padding_duration_ms, vad, frames):
"""Filters out non-voiced audio frames.
Given a webrtcvad.Vad and a source of audio frames, yields only
the voiced audio.
Uses a padded, sliding window algorithm over the audio frames.
When more than 90% of the frames in the window are voiced (as
reported by the VAD), the collector triggers and begins yielding
audio frames. Then the collector waits until 90% of the frames in
the window are unvoiced to detrigger.
The window is padded at the front and back to provide a small
amount of silence or the beginnings/endings of speech around the
voiced frames.
Arguments:
sample_rate - The audio sample rate, in Hz.
frame_duration_ms - The frame duration in milliseconds.
padding_duration_ms - The amount to pad the window, in milliseconds.
vad - An instance of webrtcvad.Vad.
frames - a source of audio frames (sequence or generator).
Returns: A generator that yields PCM audio data.
"""
num_padding_frames = int(padding_duration_ms / frame_duration_ms)
# We use a deque for our sliding window/ring buffer.
ring_buffer = collections.deque(maxlen=num_padding_frames)
# We have two states: TRIGGERED and NOTTRIGGERED. We start in the
# NOTTRIGGERED state.
triggered = False
voiced_frames = []
for frame in frames:
is_speech = vad.is_speech(frame.bytes, sample_rate)
if not triggered:
ring_buffer.append((frame, is_speech))
num_voiced = len([f for f, speech in ring_buffer if speech])
# If we're NOTTRIGGERED and more than 90% of the frames in
# the ring buffer are voiced frames, then enter the
# TRIGGERED state.
if num_voiced > 0.9 * ring_buffer.maxlen:
triggered = True
# We want to yield all the audio we see from now until
# we are NOTTRIGGERED, but we have to start with the
# audio that's already in the ring buffer.
for f, s in ring_buffer:
voiced_frames.append(f)
ring_buffer.clear()
else:
# We're in the TRIGGERED state, so collect the audio data
# and add it to the ring buffer.
voiced_frames.append(frame)
ring_buffer.append((frame, is_speech))
num_unvoiced = len([f for f, speech in ring_buffer if not speech])
# If more than 90% of the frames in the ring buffer are
# unvoiced, then enter NOTTRIGGERED and yield whatever
# audio we've collected.
if num_unvoiced > 0.9 * ring_buffer.maxlen:
triggered = False
yield b''.join([f.bytes for f in voiced_frames])
ring_buffer.clear()
voiced_frames = []
if triggered:
pass
# If we have any leftover voiced audio when we run out of input,
# yield it.
if voiced_frames:
yield b''.join([f.bytes for f in voiced_frames])

View File

@ -0,0 +1,103 @@
import glob
import webrtcvad
import logging
import wavSplit
from deepspeech import Model
from timeit import default_timer as timer
'''
Load the pre-trained model into the memory
@param models: Output Grapgh Protocol Buffer file
@param alphabet: Alphabet.txt file
@param lm: Language model file
@param trie: Trie file
@Retval
Returns a list [DeepSpeech Object, Model Load Time, LM Load Time]
'''
def load_model(models, alphabet, lm, trie):
N_FEATURES = 26
N_CONTEXT = 9
BEAM_WIDTH = 500
LM_WEIGHT = 1.75
VALID_WORD_COUNT_WEIGHT = 1.00
model_load_start = timer()
ds = Model(models, N_FEATURES, N_CONTEXT, alphabet, BEAM_WIDTH)
model_load_end = timer() - model_load_start
logging.debug("Loaded model in %0.3fs." % (model_load_end))
lm_load_start = timer()
ds.enableDecoderWithLM(alphabet, lm, trie, LM_WEIGHT, VALID_WORD_COUNT_WEIGHT)
lm_load_end = timer() - lm_load_start
logging.debug('Loaded language model in %0.3fs.' % (lm_load_end))
return [ds, model_load_end, lm_load_end]
'''
Run Inference on input audio file
@param ds: Deepspeech object
@param audio: Input audio for running inference on
@param fs: Sample rate of the input audio file
@Retval:
Returns a list [Inference, Inference Time, Audio Length]
'''
def stt(ds, audio, fs):
inference_time = 0.0
audio_length = len(audio) * (1 / 16000)
# Run Deepspeech
logging.debug('Running inference...')
inference_start = timer()
output = ds.stt(audio, fs)
inference_end = timer() - inference_start
inference_time += inference_end
logging.debug('Inference took %0.3fs for %0.3fs audio file.' % (inference_end, audio_length))
return [output, inference_time]
'''
Resolve directory path for the models and fetch each of them.
@param dirName: Path to the directory containing pre-trained models
@Retval:
Retunns a tuple containing each of the model files (pb, alphabet, lm and trie)
'''
def resolve_models(dirName):
pb = glob.glob(dirName + "/*.pb")[0]
logging.debug("Found Model: %s" % pb)
alphabet = glob.glob(dirName + "/alphabet.txt")[0]
logging.debug("Found Alphabet: %s" % alphabet)
lm = glob.glob(dirName + "/lm.binary")[0]
trie = glob.glob(dirName + "/trie")[0]
logging.debug("Found Language Model: %s" % lm)
logging.debug("Found Trie: %s" % trie)
return pb, alphabet, lm, trie
'''
Generate VAD segments. Filters out non-voiced audio frames.
@param waveFile: Input wav file to run VAD on.0
@Retval:
Returns tuple of
segments: a bytearray of multiple smaller audio frames
(The longer audio split into mutiple smaller one's)
sample_rate: Sample rate of the input audio file
audio_length: Duraton of the input audio file
'''
def vad_segment_generator(wavFile, aggressiveness):
logging.debug("Caught the wav file @: %s" % (wavFile))
audio, sample_rate, audio_length = wavSplit.read_wave(wavFile)
assert sample_rate == 16000, "Only 16000Hz input WAV files are supported for now!"
vad = webrtcvad.Vad(int(aggressiveness))
frames = wavSplit.frame_generator(30, audio, sample_rate)
frames = list(frames)
segments = wavSplit.vad_collector(sample_rate, 30, 300, vad, frames)
return segments, sample_rate, audio_length

View File

@ -0,0 +1,61 @@
## Transcribing longer audio clips
The Command and GUI tools perform transcription on long wav files.
They take in a wav file of any duration, use the WebRTC Voice Activity Detector (VAD)
to split it into smaller chunks and finally save a consolidated transcript.
### 0. Prerequisites
Setup your environment
```
~/Deepspeech$ sudo apt install virtualenv
~/Deepspeech$ cd examples/vad_transcriber
~/Deepspeech/examples/vad_transcriber$ virtualenv -p python3 venv
~/Deepspeech/examples/vad_transcriber$ source venv/bin/activate
(venv) ~/Deepspeech/examples/vad_transcriber$ pip3 install -r requirements.txt
```
### 1. Command line tool
The command line tool processes a wav file of any duration and returns a trancript
which will the saved in the same directory as the input audio file.
The command line tool gives you control over the aggressiveness of the VAD.
Set the aggressiveness mode, to an integer between 0 and 3.
0 being the least aggressive about filtering out non-speech, 3 is the most aggressive.
```
(venv) ~/Deepspeech/examples/vad_transcriber
$ python3 audioTranscript_cmd.py --aggressive 1 --audio ./audio/guido-van-rossum.wav --model ./models/0.2.0/
Filename Duration(s) Inference Time(s) Model Load Time(s) LM Load Time(s)
sample_rec.wav 13.710 20.797 5.593 17.742
```
### 2. Minimalistic GUI
The GUI tool does the same job as the CLI tool. The VAD is fixed at an aggressiveness of 1.
The output is displayed in the transcription window and saved into the directory as the input
audio file as well.
```
(venv) ~/Deepspeech/examples/vad_transcriber
$ python3 audioTranscript_gui.py
```
![Deepspeech Transcriber](../../doc/audioTranscript.png)
#### 2.1. Sporadic failures in pyqt
Some systems have encountered **_Cannot mix incompatible Qt library with this with this library_** issue.
In such a scenario, the GUI tool will not work. The following steps is known to have solved the issue in most cases
```
(venv) ~/Deepspeech/examples/vad_transcriber$ pip3 uninstall pyqt5
(venv) ~/Deepspeech/examples/vad_transcriber$ sudo apt install python3-pyqt5 canberra-gtk-module
(venv) ~/Deepspeech/examples/vad_transcriber$ export PYTHONPATH=/usr/lib/python3/dist-packages/
(venv) ~/Deepspeech/examples/vad_transcriber$ python3 audioTranscript_gui.py
```