From 56df4ebf03fbe52c78fd5b510b2e74e4613e0a0c Mon Sep 17 00:00:00 2001 From: Jordan Olafsen Date: Wed, 20 Feb 2019 21:57:27 +1300 Subject: [PATCH 1/5] Add Input Rate to examples/mic_vad_streaming. Add -r for input device sample rate and -d for device index by PyAudio --- .../mic_vad_streaming/mic_vad_streaming.py | 71 ++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/examples/mic_vad_streaming/mic_vad_streaming.py b/examples/mic_vad_streaming/mic_vad_streaming.py index 259d4fef..e6d2f775 100644 --- a/examples/mic_vad_streaming/mic_vad_streaming.py +++ b/examples/mic_vad_streaming/mic_vad_streaming.py @@ -1,6 +1,7 @@ import time, logging from datetime import datetime import threading, collections, queue, os, os.path +import audioop import wave import pyaudio import webrtcvad @@ -10,35 +11,64 @@ import numpy as np logging.basicConfig(level=20) + class Audio(object): """Streams raw audio from microphone. Data is received in a separate thread, and stored in a buffer, to be read from.""" FORMAT = pyaudio.paInt16 - RATE = 16000 + # Network/VAD rate-space + RATE_PROCESS = 16000 CHANNELS = 1 BLOCKS_PER_SECOND = 50 - BLOCK_SIZE = int(RATE / float(BLOCKS_PER_SECOND)) - def __init__(self, callback=None): + def __init__(self, callback=None, device=None, input_rate=RATE_PROCESS): def proxy_callback(in_data, frame_count, time_info, status): callback(in_data) return (None, pyaudio.paContinue) - if callback is None: callback = lambda in_data: self.buffer_queue.put(in_data) + if callback is None: + callback = lambda in_data: self.buffer_queue.put(in_data) self.buffer_queue = queue.Queue() - self.sample_rate = self.RATE - self.block_size = self.BLOCK_SIZE + self.device = device + self.input_rate = input_rate + self.sample_rate = self.RATE_PROCESS + self.block_size = int(self.RATE_PROCESS / float(self.BLOCKS_PER_SECOND)) + self.block_size_input = int(self.input_rate / float(self.BLOCKS_PER_SECOND)) self.pa = pyaudio.PyAudio() - self.stream = self.pa.open(format=self.FORMAT, - channels=self.CHANNELS, - rate=self.sample_rate, - input=True, - frames_per_buffer=self.block_size, - stream_callback=proxy_callback) + + kwargs = { + 'format': self.FORMAT, + 'channels': self.CHANNELS, + 'rate': self.input_rate, + 'input': True, + 'frames_per_buffer': self.block_size_input, + 'stream_callback': proxy_callback, + } + + # if not default device + if self.device: + kwargs['input_device_index'] = self.device + + self.stream = self.pa.open(**kwargs) self.stream.start_stream() + def resample(self, data, input_rate): + """ + Microphone may not support our native processing sampling rate, so + resample from input_rate to RATE_PROCESS here for webrtcvad and + deepspeech + + Args: + data : Input audio stream + input_rate (int): Input audio rate to resample from + """ + newfragment, state = audioop.ratecv(data, 2, 1, input_rate, + self.sample_rate, None) + return newfragment + def read(self): """Return a block of audio data, blocking if necessary.""" - return self.buffer_queue.get() + buffer = self.buffer_queue.get() + return self.resample(buffer, self.input_rate) def destroy(self): self.stream.stop_stream() @@ -58,11 +88,12 @@ class Audio(object): wf.writeframes(data) wf.close() + class VADAudio(Audio): """Filter & segment audio with voice activity detection.""" - def __init__(self, aggressiveness=3): - super().__init__() + def __init__(self, aggressiveness=3, device=None, input_rate=None): + super().__init__(device=device, input_rate=input_rate) self.vad = webrtcvad.Vad(aggressiveness) def frame_generator(self): @@ -121,7 +152,9 @@ def main(ARGS): model.enableDecoderWithLM(ARGS.alphabet, ARGS.lm, ARGS.trie, ARGS.lm_alpha, ARGS.lm_beta) # Start audio with VAD - vad_audio = VADAudio(aggressiveness=ARGS.vad_aggressiveness) + vad_audio = VADAudio(aggressiveness=ARGS.vad_aggressiveness, + device=ARGS.device, + input_rate=ARGS.rate) print("Listening (ctrl-C to exit)...") frames = vad_audio.vad_collector() @@ -146,8 +179,10 @@ def main(ARGS): print("Recognized: %s" % text) stream_context = model.setupStream() + if __name__ == '__main__': BEAM_WIDTH = 500 + DEFAULT_SAMPLE_RATE = 16000 LM_ALPHA = 0.75 LM_BETA = 1.85 N_FEATURES = 26 @@ -171,6 +206,10 @@ if __name__ == '__main__': help="Path to the language model binary file. Default: lm.binary") parser.add_argument('-t', '--trie', default='trie', help="Path to the language model trie file created with native_client/generate_trie. Default: trie") + parser.add_argument('-d', '--device', type=int, default=None, + help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index()") + parser.add_argument('-r', '--rate', type=int, default=DEFAULT_SAMPLE_RATE, + help=f"Input device sample rate. Default: {DEFAULT_SAMPLE_RATE}. Your device may require 44100.") parser.add_argument('-nf', '--n_features', type=int, default=N_FEATURES, help=f"Number of MFCC features to use. Default: {N_FEATURES}") parser.add_argument('-nc', '--n_context', type=int, default=N_CONTEXT, From dedf2911dacb0df6635e534b177240ebe5946390 Mon Sep 17 00:00:00 2001 From: Jordan Olafsen Date: Sat, 23 Feb 2019 13:35:45 +1300 Subject: [PATCH 2/5] Add resampling to microphone stream if different from processing sample rate. Uses scipy.signal --- .../mic_vad_streaming/mic_vad_streaming.py | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/examples/mic_vad_streaming/mic_vad_streaming.py b/examples/mic_vad_streaming/mic_vad_streaming.py index e6d2f775..5c385242 100644 --- a/examples/mic_vad_streaming/mic_vad_streaming.py +++ b/examples/mic_vad_streaming/mic_vad_streaming.py @@ -1,13 +1,13 @@ import time, logging from datetime import datetime import threading, collections, queue, os, os.path -import audioop -import wave -import pyaudio -import webrtcvad -from halo import Halo import deepspeech import numpy as np +import pyaudio +import wave +import webrtcvad +from halo import Halo +from scipy import signal logging.basicConfig(level=20) @@ -58,17 +58,23 @@ class Audio(object): deepspeech Args: - data : Input audio stream + data (binary): Input audio stream input_rate (int): Input audio rate to resample from """ - newfragment, state = audioop.ratecv(data, 2, 1, input_rate, - self.sample_rate, None) - return newfragment - + data16 = np.fromstring(string=data, dtype=np.int16) + resample_size = int(len(data16) / self.input_rate * self.RATE_PROCESS) + resample = signal.resample(data16, resample_size) + resample16 = np.array(resample, dtype=np.int16) + return resample16.tostring() + + def read_resampled(self): + """Return a block of audio data resampled to 16000hz, blocking if necessary.""" + return self.resample(data=self.buffer_queue.get(), + input_rate=self.input_rate) + def read(self): """Return a block of audio data, blocking if necessary.""" - buffer = self.buffer_queue.get() - return self.resample(buffer, self.input_rate) + return self.buffer_queue.get() def destroy(self): self.stream.stop_stream() @@ -98,8 +104,12 @@ class VADAudio(Audio): def frame_generator(self): """Generator that yields all audio frames from microphone.""" - while True: - yield self.read() + if self.input_rate == self.RATE_PROCESS: + while True: + yield self.read() + else: + while True: + yield self.read_resampled() def vad_collector(self, padding_ms=300, ratio=0.75, frames=None): """Generator that yields series of consecutive audio frames comprising each utterence, separated by yielding a single None. @@ -107,7 +117,8 @@ class VADAudio(Audio): Example: (frame, ..., frame, None, frame, ..., frame, None, ...) |---utterence---| |---utterence---| """ - if frames is None: frames = self.frame_generator() + if frames is None: + frames = self.frame_generator() num_padding_frames = padding_ms // self.frame_duration_ms ring_buffer = collections.deque(maxlen=num_padding_frames) triggered = False From 77b2f8f3ecaa8178d1a1c6a18457f2d5351b8678 Mon Sep 17 00:00:00 2001 From: Jordan Olafsen Date: Mon, 4 Mar 2019 23:00:00 +1300 Subject: [PATCH 3/5] Address pull request feedback from @lissyx --- examples/mic_vad_streaming/mic_vad_streaming.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) mode change 100644 => 100755 examples/mic_vad_streaming/mic_vad_streaming.py diff --git a/examples/mic_vad_streaming/mic_vad_streaming.py b/examples/mic_vad_streaming/mic_vad_streaming.py old mode 100644 new mode 100755 index 5c385242..6e7f4993 --- a/examples/mic_vad_streaming/mic_vad_streaming.py +++ b/examples/mic_vad_streaming/mic_vad_streaming.py @@ -11,7 +11,6 @@ from scipy import signal logging.basicConfig(level=20) - class Audio(object): """Streams raw audio from microphone. Data is received in a separate thread, and stored in a buffer, to be read from.""" @@ -25,8 +24,7 @@ class Audio(object): def proxy_callback(in_data, frame_count, time_info, status): callback(in_data) return (None, pyaudio.paContinue) - if callback is None: - callback = lambda in_data: self.buffer_queue.put(in_data) + if callback is None: callback = lambda in_data: self.buffer_queue.put(in_data) self.buffer_queue = queue.Queue() self.device = device self.input_rate = input_rate @@ -117,8 +115,7 @@ class VADAudio(Audio): Example: (frame, ..., frame, None, frame, ..., frame, None, ...) |---utterence---| |---utterence---| """ - if frames is None: - frames = self.frame_generator() + if frames is None: frames = self.frame_generator() num_padding_frames = padding_ms // self.frame_duration_ms ring_buffer = collections.deque(maxlen=num_padding_frames) triggered = False @@ -190,7 +187,6 @@ def main(ARGS): print("Recognized: %s" % text) stream_context = model.setupStream() - if __name__ == '__main__': BEAM_WIDTH = 500 DEFAULT_SAMPLE_RATE = 16000 @@ -218,7 +214,7 @@ if __name__ == '__main__': parser.add_argument('-t', '--trie', default='trie', help="Path to the language model trie file created with native_client/generate_trie. Default: trie") parser.add_argument('-d', '--device', type=int, default=None, - help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index()") + help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index(). If not provided, falls back to PyAudio.get_default_device()") parser.add_argument('-r', '--rate', type=int, default=DEFAULT_SAMPLE_RATE, help=f"Input device sample rate. Default: {DEFAULT_SAMPLE_RATE}. Your device may require 44100.") parser.add_argument('-nf', '--n_features', type=int, default=N_FEATURES, From ed37957865036e417efdcb189a6180c135f369e2 Mon Sep 17 00:00:00 2001 From: Carlos Fonseca M Date: Fri, 15 Mar 2019 16:14:08 -0600 Subject: [PATCH 4/5] Move NuGet builds to avoid triggered clean --- tc-tests-utils.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 1e7ac9c6..19a477a1 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -381,7 +381,7 @@ install_nuget() nuget install NAudio cp NAudio*/lib/net35/NAudio.dll ${TASKCLUSTER_TMP_DIR}/ds/ cp ${PROJECT_NAME}.${DS_VERSION}/build/libdeepspeech.so ${TASKCLUSTER_TMP_DIR}/ds/ - cp ${PROJECT_NAME}.${DS_VERSION}/lib/net462/DeepSpeechClient.dll ${TASKCLUSTER_TMP_DIR}/ds/ + cp ${PROJECT_NAME}.${DS_VERSION}/lib/net46/DeepSpeechClient.dll ${TASKCLUSTER_TMP_DIR}/ds/ ls -hal ${TASKCLUSTER_TMP_DIR}/ds/ @@ -616,21 +616,21 @@ do_deepspeech_netframework_build() /p:Configuration=Release \ /p:Platform=x64 \ /p:TargetFrameworkVersion="v4.5" \ - /p:OutputPath=bin/x64/Release/v4.5 + /p:OutputPath=bin/nuget/x64/v4.5 MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ DeepSpeechClient/DeepSpeechClient.csproj \ /p:Configuration=Release \ /p:Platform=x64 \ /p:TargetFrameworkVersion="v4.6" \ - /p:OutputPath=bin/x64/Release/v4.6 + /p:OutputPath=bin/nuget/x64/v4.6 MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ DeepSpeechClient/DeepSpeechClient.csproj \ /p:Configuration=Release \ /p:Platform=x64 \ /p:TargetFrameworkVersion="v4.7" \ - /p:OutputPath=bin/x64/Release/v4.7 + /p:OutputPath=bin/nuget/x64/v4.7 MSYS2_ARG_CONV_EXCL='/' "${MSBUILD}" \ DeepSpeechConsole/DeepSpeechConsole.csproj \ @@ -658,13 +658,13 @@ do_nuget_build() # We copy the generated clients for .NET into the Nuget framework dirs mkdir -p nupkg/lib/net45/ - cp DeepSpeechClient/bin/x64/Release/v4.5/DeepSpeechClient.dll nupkg/lib/net45/ + cp DeepSpeechClient/bin/nuget/x64/v4.5/DeepSpeechClient.dll nupkg/lib/net45/ mkdir -p nupkg/lib/net46/ - cp DeepSpeechClient/bin/x64/Release/v4.6/DeepSpeechClient.dll nupkg/lib/net46/ + cp DeepSpeechClient/bin/nuget/x64/v4.6/DeepSpeechClient.dll nupkg/lib/net46/ mkdir -p nupkg/lib/net47/ - cp DeepSpeechClient/bin/x64/Release/v4.7/DeepSpeechClient.dll nupkg/lib/net47/ + cp DeepSpeechClient/bin/nuget/x64/v4.7/DeepSpeechClient.dll nupkg/lib/net47/ PROJECT_VERSION=$(shell cat ../../../VERSION | tr -d '\n' | tr -d '\r') sed \ From 85533545f03e7d65005dcf04b0eb88a039e55ecf Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Sat, 16 Mar 2019 09:51:27 +0100 Subject: [PATCH 5/5] Remove Python 3.4 from supported training platforms --- .../test-training_upstream-linux-amd64-py34m-opt.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml diff --git a/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml deleted file mode 100644 index 6f65f902..00000000 --- a/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml +++ /dev/null @@ -1,12 +0,0 @@ -build: - template_file: test-linux-opt-base.tyml - dependencies: - - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} - args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.4.8:m" - metadata: - name: "DeepSpeech Linux AMD64 CPU upstream training Py3.4" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 3.4, CPU only, optimized version"