mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2897 from tilmankamp/live_augmentation

Browse Source 
Live augmentation
This commit is contained in:
Tilman Kamp 2020-05-19 09:05:47 +02:00 committed by GitHub
commit 1c2d89723f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1066 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
*.pyc
*.swp
*.DS_Store
*.egg-info
.pit*
/.run
/werlog.js

13
bin/build_sdb.py
View File

@ -15,7 +15,7 @@ from deepspeech_training.util.audio import (
from deepspeech_training.util.downloader import SIMPLE_BAR
from deepspeech_training.util.sample_collections import (
DirectSDBWriter,
samples_from_files,
samples_from_sources,
)
AUDIO_TYPE_LOOKUP = {"wav": AUDIO_TYPE_WAV, "opus": AUDIO_TYPE_OPUS}
@ -26,12 +26,10 @@ def build_sdb():
with DirectSDBWriter(
CLI_ARGS.target, audio_type=audio_type, labeled=not CLI_ARGS.unlabeled
) as sdb_writer:
samples = samples_from_files(CLI_ARGS.sources, labeled=not CLI_ARGS.unlabeled)
samples = samples_from_sources(CLI_ARGS.sources, labeled=not CLI_ARGS.unlabeled)
bar = progressbar.ProgressBar(max_value=len(samples), widgets=SIMPLE_BAR)
for sample in bar(
change_audio_types(
samples, audio_type=audio_type, processes=CLI_ARGS.workers
)
change_audio_types(samples, audio_type=audio_type, bitrate=CLI_ARGS.bitrate, processes=CLI_ARGS.workers)
):
sdb_writer.add(sample)
@ -55,6 +53,11 @@ def handle_args():
choices=AUDIO_TYPE_LOOKUP.keys(),
help="Audio representation inside target SDB",
)
parser.add_argument(
"--bitrate",
type=int,
help="Bitrate for lossy compressed SDB samples like in case of --audio-type opus",
)
parser.add_argument(
"--workers", type=int, default=None, help="Number of encoding SDB workers"
)

66
bin/compare_samples.py Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env python
"""
Tool for comparing two wav samples
"""
import sys
import argparse
from deepspeech_training.util.audio import AUDIO_TYPE_NP, mean_dbfs
from deepspeech_training.util.sample_collections import load_sample
def fail(message):
print(message, file=sys.stderr, flush=True)
sys.exit(1)
def compare_samples():
sample1 = load_sample(CLI_ARGS.sample1)
sample2 = load_sample(CLI_ARGS.sample2)
if sample1.audio_format != sample2.audio_format:
fail('Samples differ on: audio-format ({} and {})'.format(sample1.audio_format, sample2.audio_format))
if sample1.duration != sample2.duration:
fail('Samples differ on: duration ({} and {})'.format(sample1.duration, sample2.duration))
sample1.change_audio_type(AUDIO_TYPE_NP)
sample2.change_audio_type(AUDIO_TYPE_NP)
audio_diff = sample1.audio - sample2.audio
diff_dbfs = mean_dbfs(audio_diff)
differ_msg = 'Samples differ on: sample data ({:0.2f} dB difference) '.format(diff_dbfs)
equal_msg = 'Samples are considered equal ({:0.2f} dB difference)'.format(diff_dbfs)
if CLI_ARGS.if_differ:
if diff_dbfs <= CLI_ARGS.threshold:
fail(equal_msg)
if not CLI_ARGS.no_success_output:
print(differ_msg, file=sys.stderr, flush=True)
else:
if diff_dbfs > CLI_ARGS.threshold:
fail(differ_msg)
if not CLI_ARGS.no_success_output:
print(equal_msg, file=sys.stderr, flush=True)
def handle_args():
parser = argparse.ArgumentParser(
description="Tool for checking similarity of two samples"
)
parser.add_argument("sample1", help="Filename of sample 1 to compare")
parser.add_argument("sample2", help="Filename of sample 2 to compare")
parser.add_argument("--threshold", type=float, default=-60.0,
help="dB of sample deltas above which they are considered different")
parser.add_argument(
"--if-differ",
action="store_true",
help="If to succeed and return status code 0 on different signals and fail on equal ones (inverse check)."
"This will still fail on different formats or durations.",
)
parser.add_argument(
"--no-success-output",
action="store_true",
help="Stay silent on success (if samples are equal of - with --if-differ - samples are not equal)",
)
return parser.parse_args()
if __name__ == "__main__":
CLI_ARGS = handle_args()
compare_samples()

110
bin/play.py
View File

@ -1,54 +1,72 @@
#!/usr/bin/env python
"""
Tool for playing samples from Sample Databases (SDB files) and DeepSpeech CSV files
Tool for playing (and augmenting) single samples or samples from Sample Databases (SDB files) and DeepSpeech CSV files
Use "python3 build_sdb.py -h" for help
"""
import argparse
import random
import os
import sys
import random
import argparse
from deepspeech_training.util.audio import AUDIO_TYPE_PCM
from deepspeech_training.util.sample_collections import LabeledSample, samples_from_file
from deepspeech_training.util.audio import LOADABLE_AUDIO_EXTENSIONS, AUDIO_TYPE_PCM, AUDIO_TYPE_WAV
from deepspeech_training.util.sample_collections import SampleList, LabeledSample, samples_from_source, augment_samples
def play_sample(samples, index):
if index < 0:
index = len(samples) + index
if CLI_ARGS.random:
index = random.randint(0, len(samples))
elif index >= len(samples):
print("No sample with index {}".format(CLI_ARGS.start))
sys.exit(1)
sample = samples[index]
print('Sample "{}"'.format(sample.sample_id))
if isinstance(sample, LabeledSample):
print(' "{}"'.format(sample.transcript))
sample.change_audio_type(AUDIO_TYPE_PCM)
rate, channels, width = sample.audio_format
wave_obj = simpleaudio.WaveObject(sample.audio, channels, width, rate)
play_obj = wave_obj.play()
play_obj.wait_done()
def play_collection():
samples = samples_from_file(CLI_ARGS.collection, buffering=0)
def get_samples_in_play_order():
ext = os.path.splitext(CLI_ARGS.source)[1].lower()
if ext in LOADABLE_AUDIO_EXTENSIONS:
samples = SampleList([(CLI_ARGS.source, 0)], labeled=False)
else:
samples = samples_from_source(CLI_ARGS.source, buffering=0)
played = 0
index = CLI_ARGS.start
while True:
if 0 <= CLI_ARGS.number <= played:
return
play_sample(samples, index)
if CLI_ARGS.random:
yield samples[random.randint(0, len(samples) - 1)]
elif index < 0:
yield samples[len(samples) + index]
elif index >= len(samples):
print("No sample with index {}".format(CLI_ARGS.start))
sys.exit(1)
else:
yield samples[index]
played += 1
index = (index + 1) % len(samples)
def play_collection():
samples = get_samples_in_play_order()
samples = augment_samples(samples,
audio_type=AUDIO_TYPE_PCM,
augmentation_specs=CLI_ARGS.augment,
process_ahead=0,
fixed_clock=CLI_ARGS.clock)
for sample in samples:
if not CLI_ARGS.quiet:
print('Sample "{}"'.format(sample.sample_id), file=sys.stderr)
if isinstance(sample, LabeledSample):
print(' "{}"'.format(sample.transcript), file=sys.stderr)
if CLI_ARGS.pipe:
sample.change_audio_type(AUDIO_TYPE_WAV)
sys.stdout.buffer.write(sample.audio.getvalue())
return
wave_obj = simpleaudio.WaveObject(sample.audio,
sample.audio_format.channels,
sample.audio_format.width,
sample.audio_format.rate)
play_obj = wave_obj.play()
play_obj.wait_done()
def handle_args():
parser = argparse.ArgumentParser(
description="Tool for playing samples from Sample Databases (SDB files) "
description="Tool for playing (and augmenting) single samples or samples from Sample Databases (SDB files) "
"and DeepSpeech CSV files"
)
parser.add_argument("collection", help="Sample DB or CSV file to play samples from")
parser.add_argument("source", help="Sample DB, CSV or WAV file to play samples from")
parser.add_argument(
"--start",
type=int,
@ -66,16 +84,40 @@ def handle_args():
action="store_true",
help="If samples should be played in random order",
)
parser.add_argument(
"--augment",
action='append',
help="Add an augmentation operation",
)
parser.add_argument(
"--clock",
type=float,
default=0.5,
help="Simulates clock value used for augmentations during training."
"Ranges from 0.0 (representing parameter start values) to"
"1.0 (representing parameter end values)",
)
parser.add_argument(
"--pipe",
action="store_true",
help="Pipe first sample as wav file to stdout. Forces --number to 1.",
)
parser.add_argument(
"--quiet",
action="store_true",
help="No info logging to console",
)
return parser.parse_args()
if __name__ == "__main__":
try:
import simpleaudio
except ModuleNotFoundError:
print('play.py requires Python package "simpleaudio"')
sys.exit(1)
CLI_ARGS = handle_args()
if not CLI_ARGS.pipe:
try:
import simpleaudio
except ModuleNotFoundError:
print('Unless using the --pipe flag, play.py requires Python package "simpleaudio" for playing samples')
sys.exit(1)
try:
play_collection()
except KeyboardInterrupt:

66
bin/run-tc-signal_augmentations.sh Executable file
View File

@ -0,0 +1,66 @@
#!/bin/sh
set -xe
ldc93s1_dir=`cd data/smoke_test; pwd`
ldc93s1_csv="${ldc93s1_dir}/LDC93S1.csv"
ldc93s1_wav="${ldc93s1_dir}/LDC93S1.wav"
ldc93s1_overlay_csv="${ldc93s1_dir}/LDC93S1_overlay.csv"
ldc93s1_overlay_wav="${ldc93s1_dir}/LDC93S1_reversed.wav"
play="python bin/play.py --number 1 --quiet"
compare="python bin/compare_samples.py --no-success-output"
if [ ! -f "${ldc93s1_csv}" ]; then
echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}."
python -u bin/import_ldc93s1.py ${ldc93s1_dir}
fi;
if [ ! -f "${ldc93s1_overlay_csv}" ]; then
echo "Reversing ${ldc93s1_wav} to ${ldc93s1_overlay_wav}."
sox "${ldc93s1_wav}" "${ldc93s1_overlay_wav}" reverse
echo "Creating ${ldc93s1_overlay_csv}."
printf "wav_filename\n${ldc93s1_overlay_wav}" > "${ldc93s1_overlay_csv}"
fi;
if ! $compare --if-differ "${ldc93s1_wav}" "${ldc93s1_overlay_wav}"; then
echo "Sample comparison tool not working correctly"
exit 1
fi
$play ${ldc93s1_wav} --augment overlay[source="${ldc93s1_overlay_csv}",snr=20] --pipe >/tmp/overlay-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/overlay-test.wav; then
echo "Overlay augmentation had no effect or changed basic sample properties"
exit 1
fi
$play ${ldc93s1_wav} --augment reverb[delay=50.0,decay=2.0] --pipe >/tmp/reverb-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/reverb-test.wav; then
echo "Reverb augmentation had no effect or changed basic sample properties"
exit 1
fi
$play ${ldc93s1_wav} --augment gaps[n=10,size=100.0] --pipe >/tmp/gaps-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/gaps-test.wav; then
echo "Gaps augmentation had no effect or changed basic sample properties"
exit 1
fi
$play ${ldc93s1_wav} --augment resample[rate=4000] --pipe >/tmp/resample-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/resample-test.wav; then
echo "Resample augmentation had no effect or changed basic sample properties"
exit 1
fi
$play ${ldc93s1_wav} --augment codec[bitrate=4000] --pipe >/tmp/codec-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/codec-test.wav; then
echo "Codec augmentation had no effect or changed basic sample properties"
exit 1
fi
$play ${ldc93s1_wav} --augment volume --pipe >/tmp/volume-test.wav
if ! $compare --if-differ "${ldc93s1_wav}" /tmp/volume-test.wav; then
echo "Volume augmentation had no effect or changed basic sample properties"
exit 1
fi

130
doc/TRAINING.rst
View File

@ -263,21 +263,140 @@ UTF-8 mode
DeepSpeech includes a UTF-8 operating mode which can be useful to model languages with very large alphabets, such as Chinese Mandarin. For details on how it works and how to use it, see :ref:`decoder-docs`.
Training with augmentation
^^^^^^^^^^^^^^^^^^^^^^^^^^
Augmentation
^^^^^^^^^^^^
Augmentation is a useful technique for better generalization of machine learning models. Thus, a pre-processing pipeline with various augmentation techniques on raw pcm and spectrogram has been implemented and can be used while training the model. Following are the available augmentation techniques that can be enabled at training time by using the corresponding flags in the command line.
Audio Augmentation
~~~~~~~~~~~~~~~~~~
Audio Augmentation
------------------
Augmentations that are applied before potential feature caching can be specified through the ``--augment`` flag. Being a multi-flag, it can be specified multiple times (see below for an example).
Each sample of the training data will get treated by every specified augmentation in their given order. However: whether an augmentation will actually get applied to a sample is decided by chance on base of the augmentation's probability value. For example a value of ``p=0.1`` would apply the according augmentation to just 10% of all samples. This also means that augmentations are not mutually exclusive on a per-sample basis.
The ``--augment`` flag uses a common syntax for all augmentation types: ``--augment augmentation_type1[param1=value1,param2=value2,...] --augment augmentation_type2[param1=value1,param2=value2,...] ...``. For example, for the ``overlay`` augmentation:
.. code-block:: bash
python3 DeepSpeech.py --augment overlay[p=0.1,source=/path/to/audio.sdb,snr=20.0] ...
In the documentation below, whenever a value is specified as ``<float-range>`` or ``<int-range>``, it supports one of the follow formats:
* ``<value>``: A constant (int or float) value.
* ``<value>~<r>``: A center value with a randomization radius around it. E.g. ``1.2~0.4`` will result in picking of a uniformly random value between 0.8 and 1.6 on each sample augmentation.
* ``<start>:<end>``: The value will range from `<start>` at the beginning of an epoch to `<end>` at the end of an epoch. E.g. ``-0.2:1.2`` (float) or ``2000:4000`` (int)
* ``<start>:<end>~<r>``: Combination of the two previous cases with a ranging center value. E.g. ``4-6~2`` would at the beginning of an epoch pick values between 2 and 6 and at the end of an epoch between 4 and 8.
Ranges specified with integer limits will only assume integer (rounded) values.
If feature caching is enabled, these augmentations will only be performed on the first epoch and the result will be reused for subsequent epochs. The flag ``--augmentations_per_epoch N`` (by default `N` is 1) could be used to get more than one epoch worth of augmentations into the cache. During training, each epoch will do ``N`` passes over the training set, each time performing augmentation independently of previous passes. Be aware: this will also multiply the required size of the feature cache if it's enabled.
**Overlay augmentation** ``--augment overlay[p=<float>,source=<str>,snr=<float-range>,layers=<int-range>]``
Layers another audio source (multiple times) onto augmented samples.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **source**: path to the sample collection to use for augmenting (*.sdb or *.csv file). It will be repeated if there are not enough samples left.
* **snr**: signal to noise ratio in dB - positive values for lowering volume of the overlay in relation to the sample
* **layers**: number of layers added onto the sample (e.g. 10 layers of speech to get "cocktail-party effect"). A layer is just a sample of the same duration as the sample to augment. It gets stitched together from as many source samples as required.
**Reverb augmentation** ``--augment reverb[p=<float>,delay=<float-range>,decay=<float-range>]``
Adds simplified (no all-pass filters) `Schroeder reverberation <https://ccrma.stanford.edu/~jos/pasp/Schroeder_Reverberators.html>`_ to the augmented samples.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **delay**: time delay in ms for the first signal reflection - higher values are widening the perceived "room"
* **decay**: sound decay in dB per reflection - higher values will result in a less reflective perceived "room"
**Gaps augmentation** ``--augment gaps[p=<float>,n=<int-range>,size=<float-range>]``
Sets time-intervals within the augmented samples to zero (silence) at random positions.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **n**: number of intervals to set to zero
* **size**: duration of intervals in ms
**Resample augmentation** ``--augment resample[p=<float>,rate=<int-range>]``
Resamples augmented samples to another sample rate and then resamples back to the original sample rate.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **rate**: sample-rate to re-sample to
**Codec augmentation** ``--augment codec[p=<float>,bitrate=<int-range>]``
Compresses and then decompresses augmented samples using the lossy Opus audio codec.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **bitrate**: bitrate used during compression
**Volume augmentation** ``--augment volume[p=<float>,dbfs=<float-range>]``
Measures and levels augmented samples to a target dBFS value.
* **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method
* **dbfs** : target volume in dBFS (default value of 3.0103 will normalize min and max amplitudes to -1.0/1.0)
Example training with all augmentations:
.. code-block:: bash
python -u DeepSpeech.py \
--train_files "train.sdb" \
--augmentations_per_epoch 10 \
--augment overlay[p=0.5,source=noise.sdb,layers=1,snr=50:20~10] \
--augment overlay[p=0.2,source=voices.sdb,layers=10:6,snr=50:20~10] \
--augment reverb[p=0.1,delay=50.0~30.0,decay=10.0:2.0~1.0] \
--augment gaps[p=0.05,n=1:3~2,size=10:100] \
--augment resample[p=0.1,rate=12000:8000~4000] \
--augment codec[p=0.1,bitrate=48000:16000] \
--augment volume[p=0.1,dbfs=-10:-40] \
[...]
The ``bin/play.py`` tool also supports ``--augment`` parameters and can be used for experimenting with different configurations.
Example of playing all samples with reverberation and maximized volume:
.. code-block:: bash
bin/play.py --augment reverb[p=0.1,delay=50.0,decay=2.0] --augment volume --random test.sdb
Example simulation of the codec augmentation of a wav-file first at the beginning and then at the end of an epoch:
.. code-block:: bash
bin/play.py --augment codec[p=0.1,bitrate=48000:16000] --clock 0.0 test.wav
bin/play.py --augment codec[p=0.1,bitrate=48000:16000] --clock 1.0 test.wav
The following augmentations are applied after feature caching, hence the way they are applied will not repeat epoch-wise.
Working on spectrogram and feature level, `bin/play.py` offers no ability to simulate them.
#. **Standard deviation for Gaussian additive noise:** ``--data_aug_features_additive``
#. **Standard deviation for Normal distribution around 1 for multiplicative noise:** ``--data_aug_features_multiplicative``
#. **Standard deviation for speeding-up tempo. If Standard deviation is 0, this augmentation is not performed:** ``--augmentation_speed_up_std``
Spectrogram Augmentation
~~~~~~~~~~~~~~~~~~~~~~~~
------------------------
Inspired by Google Paper on `SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition <https://arxiv.org/abs/1904.08779>`_
@ -306,4 +425,3 @@ Inspired by Google Paper on `SpecAugment: A Simple Data Augmentation Method for
* Min value of pitch scaling: ``--augmentation_pitch_and_tempo_scaling_min_pitch eg:0.95``
* Max value of pitch scaling: ``--augmentation_pitch_and_tempo_scaling_max_pitch eg:1.2``
* Max value of tempo scaling: ``--augmentation_pitch_and_tempo_scaling_max_tempo eg:1.2``

28
taskcluster/tc-signal_augmentation-tests.sh Normal file
View File

@ -0,0 +1,28 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
ds=$2
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
mkdir -p /tmp/train || true
mkdir -p /tmp/train_tflite || true
virtualenv_activate "${pyalias}" "deepspeech"
set -o pipefail
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
pushd ${HOME}/DeepSpeech/ds
pip install --upgrade . | cat
popd
set +o pipefail
pushd ${HOME}/DeepSpeech/ds/
time ./bin/run-tc-signal_augmentations.sh
popd
virtualenv_deactivate "${pyalias}" "deepspeech"

12
taskcluster/test-signal_augmentations-linux-amd64-py36m-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-signal_augmentation-tests.sh 3.6.10:m"
metadata:
name: "DeepSpeech Linux AMD64 CPU signal augmentations Py3.6"
description: "Augmenting LDC93S1 sample in different ways for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version"

114
tests/test_value_range.py Normal file
View File

@ -0,0 +1,114 @@
import unittest
from deepspeech_training.util.helpers import ValueRange, get_value_range, pick_value_from_range
class TestValueRange(unittest.TestCase):
def _ending_tester(self, value, value_type, expected):
result = get_value_range(value, value_type)
self.assertEqual(result, expected)
def test_int_str_scalar(self):
self._ending_tester('1', int, ValueRange(1, 1, 0))
def test_int_str_scalar_radius(self):
self._ending_tester('1~3', int, ValueRange(1, 1, 3))
def test_int_str_range(self):
self._ending_tester('1:2', int, ValueRange(1, 2, 0))
def test_int_str_range_radius(self):
self._ending_tester('1:2~3', int, ValueRange(1, 2, 3))
def test_int_scalar(self):
self._ending_tester(1, int, ValueRange(1, 1, 0))
def test_int_2tuple(self):
self._ending_tester((1, 2), int, ValueRange(1, 2, 0))
def test_int_3tuple(self):
self._ending_tester((1, 2, 3), int, ValueRange(1, 2, 3))
def test_float_str_scalar(self):
self._ending_tester('1.0', float, ValueRange(1.0, 1.0, 0.0))
def test_float_str_scalar_radius(self):
self._ending_tester('1.0~3.0', float, ValueRange(1.0, 1.0, 3.0))
def test_float_str_range(self):
self._ending_tester('1.0:2.0', float, ValueRange(1.0, 2.0, 0.0))
def test_float_str_range_radius(self):
self._ending_tester('1.0:2.0~3.0', float, ValueRange(1.0, 2.0, 3.0))
def test_float_scalar(self):
self._ending_tester(1.0, float, ValueRange(1.0, 1.0, 0.0))
def test_float_2tuple(self):
self._ending_tester((1.0, 2.0), float, ValueRange(1.0, 2.0, 0.0))
def test_float_3tuple(self):
self._ending_tester((1.0, 2.0, 3.0), float, ValueRange(1.0, 2.0, 3.0))
def test_float_int_3tuple(self):
self._ending_tester((1, 2, 3), float, ValueRange(1.0, 2.0, 3.0))
class TestPickValueFromFixedRange(unittest.TestCase):
def _ending_tester(self, value_range, clock, expected):
is_int = isinstance(value_range.start, int)
result = pick_value_from_range(value_range, clock)
self.assertEqual(result, expected)
self.assertTrue(isinstance(result, int if is_int else float))
def test_int_0(self):
self._ending_tester(ValueRange(1, 3, 0), 0.0, 1)
def test_int_half(self):
self._ending_tester(ValueRange(1, 3, 0), 0.5, 2)
def test_int_1(self):
self._ending_tester(ValueRange(1, 3, 0), 1.0, 3)
def test_float_0(self):
self._ending_tester(ValueRange(1.0, 2.0, 0.0), 0.0, 1.0)
def test_float_half(self):
self._ending_tester(ValueRange(1.0, 2.0, 0.0), 0.5, 1.5)
def test_float_1(self):
self._ending_tester(ValueRange(1.0, 2.0, 0.0), 1.0, 2.0)
class TestPickValueFromRandomizedRange(unittest.TestCase):
def _ending_tester(self, value_range, clock, expected_min, expected_max):
is_int = isinstance(value_range.start, int)
results = list(map(lambda x: pick_value_from_range(value_range, clock), range(100)))
self.assertGreater(len(set(results)), 80)
self.assertTrue(all(map(lambda x: expected_min <= x <= expected_max, results)))
self.assertTrue(all(map(lambda x: isinstance(x, int if is_int else float), results)))
def test_int_0(self):
self._ending_tester(ValueRange(10000, 30000, 10000), 0.0, 0, 20000)
def test_int_half(self):
self._ending_tester(ValueRange(10000, 30000, 10000), 0.5, 10000, 30000)
def test_int_1(self):
self._ending_tester(ValueRange(10000, 30000, 10000), 1.0, 20000, 40000)
def test_float_0(self):
self._ending_tester(ValueRange(10000.0, 30000.0, 10000.0), 0.0, 0.0, 20000.0)
def test_float_half(self):
self._ending_tester(ValueRange(10000.0, 30000.0, 10000.0), 0.5, 10000.0, 30000.0)
def test_float_1(self):
self._ending_tester(ValueRange(10000.0, 30000.0, 10000.0), 1.0, 20000.0, 40000.0)
if __name__ == '__main__':
unittest.main()

2
training/deepspeech_training/train.py
View File

@ -424,6 +424,8 @@ def train():
# Create training and validation datasets
train_set = create_dataset(FLAGS.train_files.split(','),
batch_size=FLAGS.train_batch_size,
repetitions=FLAGS.augmentations_per_epoch,
augmentation_specs=FLAGS.augment,
enable_cache=FLAGS.feature_cache and do_cache_dataset,
cache_path=FLAGS.feature_cache,
train_phase=True,

153
training/deepspeech_training/util/audio.py
View File

@ -1,22 +1,27 @@
import os
import io
import wave
import math
import tempfile
import collections
import numpy as np
from .helpers import LimitingPool
from collections import namedtuple
AudioFormat = namedtuple('AudioFormat', 'rate channels width')
DEFAULT_RATE = 16000
DEFAULT_CHANNELS = 1
DEFAULT_WIDTH = 2
DEFAULT_FORMAT = (DEFAULT_RATE, DEFAULT_CHANNELS, DEFAULT_WIDTH)
DEFAULT_FORMAT = AudioFormat(DEFAULT_RATE, DEFAULT_CHANNELS, DEFAULT_WIDTH)
AUDIO_TYPE_NP = 'application/vnd.mozilla.np'
AUDIO_TYPE_PCM = 'application/vnd.mozilla.pcm'
AUDIO_TYPE_WAV = 'audio/wav'
AUDIO_TYPE_OPUS = 'application/vnd.mozilla.opus'
SERIALIZABLE_AUDIO_TYPES = [AUDIO_TYPE_WAV, AUDIO_TYPE_OPUS]
LOADABLE_AUDIO_EXTENSIONS = {'.wav': AUDIO_TYPE_WAV}
OPUS_PCM_LEN_SIZE = 4
OPUS_RATE_SIZE = 4
@ -33,7 +38,7 @@ class Sample:
----------
audio_type : str
See `__init__`.
audio_format : tuple:(int, int, int)
audio_format : util.audio.AudioFormat
See `__init__`.
audio : binary
Audio data represented as indicated by `audio_type`
@ -55,8 +60,7 @@ class Sample:
raw_data : binary
Audio data in the form of the provided representation type (see audio_type).
For types util.audio.AUDIO_TYPE_OPUS or util.audio.AUDIO_TYPE_WAV data can also be passed as a bytearray.
audio_format : tuple
Tuple of sample-rate, number of channels and sample-width.
audio_format : util.audio.AudioFormat
Required in case of audio_type = util.audio.AUDIO_TYPE_PCM or util.audio.AUDIO_TYPE_NP,
as this information cannot be derived from raw audio data.
sample_id : str
@ -79,7 +83,7 @@ class Sample:
else:
raise ValueError('Unsupported audio type: {}'.format(self.audio_type))
def change_audio_type(self, new_audio_type):
def change_audio_type(self, new_audio_type, bitrate=None):
"""
In-place conversion of audio data into a different representation.
@ -87,7 +91,8 @@ class Sample:
----------
new_audio_type : str
New audio-type - see `__init__`.
Not supported: Converting from AUDIO_TYPE_NP into any other type.
bitrate : int
Bitrate to use in case of converting to a lossy audio-type.
"""
if self.audio_type == new_audio_type:
return
@ -95,13 +100,15 @@ class Sample:
self.audio_format, audio = read_audio(self.audio_type, self.audio)
self.audio.close()
self.audio = audio
elif new_audio_type == AUDIO_TYPE_PCM and self.audio_type == AUDIO_TYPE_NP:
self.audio = np_to_pcm(self.audio, self.audio_format)
elif new_audio_type == AUDIO_TYPE_NP:
self.change_audio_type(AUDIO_TYPE_PCM)
self.audio = pcm_to_np(self.audio_format, self.audio)
self.audio = pcm_to_np(self.audio, self.audio_format)
elif new_audio_type in SERIALIZABLE_AUDIO_TYPES:
self.change_audio_type(AUDIO_TYPE_PCM)
audio_bytes = io.BytesIO()
write_audio(new_audio_type, audio_bytes, self.audio_format, self.audio)
write_audio(new_audio_type, audio_bytes, self.audio, audio_format=self.audio_format, bitrate=bitrate)
audio_bytes.seek(0)
self.audio = audio_bytes
else:
@ -111,40 +118,47 @@ class Sample:
def _change_audio_type(sample_and_audio_type):
sample, audio_type = sample_and_audio_type
sample.change_audio_type(audio_type)
sample, audio_type, bitrate = sample_and_audio_type
sample.change_audio_type(audio_type, bitrate=bitrate)
return sample
def change_audio_types(samples, audio_type=AUDIO_TYPE_PCM, processes=None, process_ahead=None):
def change_audio_types(samples, audio_type=AUDIO_TYPE_PCM, bitrate=None, processes=None, process_ahead=None):
with LimitingPool(processes=processes, process_ahead=process_ahead) as pool:
yield from pool.imap(_change_audio_type, map(lambda s: (s, audio_type), samples))
yield from pool.imap(_change_audio_type, map(lambda s: (s, audio_type, bitrate), samples))
def get_audio_type_from_extension(ext):
if ext in LOADABLE_AUDIO_EXTENSIONS:
return LOADABLE_AUDIO_EXTENSIONS[ext]
return None
def read_audio_format_from_wav_file(wav_file):
return wav_file.getframerate(), wav_file.getnchannels(), wav_file.getsampwidth()
return AudioFormat(wav_file.getframerate(), wav_file.getnchannels(), wav_file.getsampwidth())
def get_num_samples(pcm_buffer_size, audio_format=DEFAULT_FORMAT):
_, channels, width = audio_format
return pcm_buffer_size // (channels * width)
return pcm_buffer_size // (audio_format.channels * audio_format.width)
def get_pcm_duration(pcm_buffer_size, audio_format=DEFAULT_FORMAT):
"""Calculates duration in seconds of a binary PCM buffer (typically read from a WAV file)"""
return get_num_samples(pcm_buffer_size, audio_format) / audio_format[0]
return get_num_samples(pcm_buffer_size, audio_format) / audio_format.rate
def get_np_duration(np_len, audio_format=DEFAULT_FORMAT):
"""Calculates duration in seconds of NumPy audio data"""
return np_len / audio_format[0]
return np_len / audio_format.rate
def convert_audio(src_audio_path, dst_audio_path, file_type=None, audio_format=DEFAULT_FORMAT):
sample_rate, channels, width = audio_format
import sox
transformer = sox.Transformer()
transformer.set_output_format(file_type=file_type, rate=sample_rate, channels=channels, bits=width*8)
transformer.set_output_format(file_type=file_type,
rate=audio_format.rate,
channels=audio_format.channels,
bits=audio_format.width * 8)
transformer.build(src_audio_path, dst_audio_path)
@ -181,7 +195,7 @@ class AudioFile:
def read_frames(wav_file, frame_duration_ms=30, yield_remainder=False):
audio_format = read_audio_format_from_wav_file(wav_file)
frame_size = int(audio_format[0] * (frame_duration_ms / 1000.0))
frame_size = int(audio_format.rate * (frame_duration_ms / 1000.0))
while True:
try:
data = wav_file.readframes(frame_size)
@ -203,13 +217,12 @@ def vad_split(audio_frames,
num_padding_frames=10,
threshold=0.5,
aggressiveness=3):
from webrtcvad import Vad
sample_rate, channels, width = audio_format
if channels != 1:
from webrtcvad import Vad # pylint: disable=import-outside-toplevel
if audio_format.channels != 1:
raise ValueError('VAD-splitting requires mono samples')
if width != 2:
if audio_format.width != 2:
raise ValueError('VAD-splitting requires 16 bit samples')
if sample_rate not in [8000, 16000, 32000, 48000]:
if audio_format.rate not in [8000, 16000, 32000, 48000]:
raise ValueError('VAD-splitting only supported for sample rates 8000, 16000, 32000, or 48000')
if aggressiveness not in [0, 1, 2, 3]:
raise ValueError('VAD-splitting aggressiveness mode has to be one of 0, 1, 2, or 3')
@ -223,7 +236,7 @@ def vad_split(audio_frames,
frame_duration_ms = get_pcm_duration(len(frame), audio_format) * 1000
if int(frame_duration_ms) not in [10, 20, 30]:
raise ValueError('VAD-splitting only supported for frame durations 10, 20, or 30 ms')
is_speech = vad.is_speech(frame, sample_rate)
is_speech = vad.is_speech(frame, audio_format.rate)
if not triggered:
ring_buffer.append((frame, is_speech))
num_voiced = len([f for f, speech in ring_buffer if speech])
@ -261,21 +274,22 @@ def get_opus_frame_size(rate):
return 60 * rate // 1000
def write_opus(opus_file, audio_format, audio_data):
rate, channels, width = audio_format
frame_size = get_opus_frame_size(rate)
def write_opus(opus_file, audio_data, audio_format=DEFAULT_FORMAT, bitrate=None):
frame_size = get_opus_frame_size(audio_format.rate)
import opuslib # pylint: disable=import-outside-toplevel
encoder = opuslib.Encoder(rate, channels, 'audio')
chunk_size = frame_size * channels * width
encoder = opuslib.Encoder(audio_format.rate, audio_format.channels, 'audio')
if bitrate is not None:
encoder.bitrate = bitrate
chunk_size = frame_size * audio_format.channels * audio_format.width
opus_file.write(pack_number(len(audio_data), OPUS_PCM_LEN_SIZE))
opus_file.write(pack_number(rate, OPUS_RATE_SIZE))
opus_file.write(pack_number(channels, OPUS_CHANNELS_SIZE))
opus_file.write(pack_number(width, OPUS_WIDTH_SIZE))
opus_file.write(pack_number(audio_format.rate, OPUS_RATE_SIZE))
opus_file.write(pack_number(audio_format.channels, OPUS_CHANNELS_SIZE))
opus_file.write(pack_number(audio_format.width, OPUS_WIDTH_SIZE))
for i in range(0, len(audio_data), chunk_size):
chunk = audio_data[i:i + chunk_size]
# Preventing non-deterministic encoding results from uninitialized remainder of the encoder buffer
if len(chunk) < chunk_size:
chunk = chunk + bytearray(chunk_size - len(chunk))
chunk = chunk + b'\0' * (chunk_size - len(chunk))
encoded = encoder.encode(chunk, frame_size)
opus_file.write(pack_number(len(encoded), OPUS_CHUNK_LEN_SIZE))
opus_file.write(encoded)
@ -287,15 +301,14 @@ def read_opus_header(opus_file):
rate = unpack_number(opus_file.read(OPUS_RATE_SIZE))
channels = unpack_number(opus_file.read(OPUS_CHANNELS_SIZE))
width = unpack_number(opus_file.read(OPUS_WIDTH_SIZE))
return pcm_buffer_size, (rate, channels, width)
return pcm_buffer_size, AudioFormat(rate, channels, width)
def read_opus(opus_file):
pcm_buffer_size, audio_format = read_opus_header(opus_file)
rate, channels, _ = audio_format
frame_size = get_opus_frame_size(rate)
frame_size = get_opus_frame_size(audio_format.rate)
import opuslib # pylint: disable=import-outside-toplevel
decoder = opuslib.Decoder(rate, channels)
decoder = opuslib.Decoder(audio_format.rate, audio_format.channels)
audio_data = bytearray()
while len(audio_data) < pcm_buffer_size:
chunk_len = unpack_number(opus_file.read(OPUS_CHUNK_LEN_SIZE))
@ -303,15 +316,14 @@ def read_opus(opus_file):
decoded = decoder.decode(chunk, frame_size)
audio_data.extend(decoded)
audio_data = audio_data[:pcm_buffer_size]
return audio_format, audio_data
return audio_format, bytes(audio_data)
def write_wav(wav_file, audio_format, pcm_data):
def write_wav(wav_file, pcm_data, audio_format=DEFAULT_FORMAT):
with wave.open(wav_file, 'wb') as wav_file_writer:
rate, channels, width = audio_format
wav_file_writer.setframerate(rate)
wav_file_writer.setnchannels(channels)
wav_file_writer.setsampwidth(width)
wav_file_writer.setframerate(audio_format.rate)
wav_file_writer.setnchannels(audio_format.channels)
wav_file_writer.setsampwidth(audio_format.width)
wav_file_writer.writeframes(pcm_data)
@ -331,11 +343,11 @@ def read_audio(audio_type, audio_file):
raise ValueError('Unsupported audio type: {}'.format(audio_type))
def write_audio(audio_type, audio_file, audio_format, pcm_data):
def write_audio(audio_type, audio_file, pcm_data, audio_format=DEFAULT_FORMAT, bitrate=None):
if audio_type == AUDIO_TYPE_WAV:
return write_wav(audio_file, audio_format, pcm_data)
return write_wav(audio_file, pcm_data, audio_format=audio_format)
if audio_type == AUDIO_TYPE_OPUS:
return write_opus(audio_file, audio_format, pcm_data)
return write_opus(audio_file, pcm_data, audio_format=audio_format, bitrate=bitrate)
raise ValueError('Unsupported audio type: {}'.format(audio_type))
@ -358,12 +370,45 @@ def read_duration(audio_type, audio_file):
raise ValueError('Unsupported audio type: {}'.format(audio_type))
def pcm_to_np(audio_format, pcm_data):
_, channels, width = audio_format
if width not in [1, 2, 4]:
raise ValueError('Unsupported sample width: {}'.format(width))
dtype = [None, np.int8, np.int16, None, np.int32][width]
def get_dtype(audio_format):
if audio_format.width not in [1, 2, 4]:
raise ValueError('Unsupported sample width: {}'.format(audio_format.width))
return [None, np.int8, np.int16, None, np.int32][audio_format.width]
def pcm_to_np(pcm_data, audio_format=DEFAULT_FORMAT):
assert audio_format.channels == 1 # only mono supported for now
dtype = get_dtype(audio_format)
samples = np.frombuffer(pcm_data, dtype=dtype)
assert channels == 1 # only mono supported for now
samples = samples.astype(np.float32) / np.iinfo(dtype).max
return np.expand_dims(samples, axis=1)
def np_to_pcm(np_data, audio_format=DEFAULT_FORMAT):
assert audio_format.channels == 1 # only mono supported for now
dtype = get_dtype(audio_format)
np_data = np_data.squeeze()
np_data = np_data * np.iinfo(dtype).max
np_data = np_data.astype(dtype)
return np_data.tobytes()
def rms_to_dbfs(rms):
return 20.0 * math.log10(max(1e-16, rms)) + 3.0103
def max_dbfs(sample_data):
# Peak dBFS based on the maximum energy sample. Will prevent overdrive if used for normalization.
return rms_to_dbfs(max(abs(np.min(sample_data)), abs(np.max(sample_data))))
def mean_dbfs(sample_data):
return rms_to_dbfs(math.sqrt(np.mean(np.square(sample_data, dtype=np.float64))))
def gain_db_to_ratio(gain_db):
return math.pow(10.0, gain_db / 20.0)
def normalize_audio(sample_data, dbfs=3.0103):
return np.maximum(np.minimum(sample_data * gain_db_to_ratio(dbfs - max_dbfs(sample_data)), 1.0), -1.0)

21
training/deepspeech_training/util/feeding.py
View File

@ -12,8 +12,8 @@ from .config import Config
from .text import text_to_char_array
from .flags import FLAGS
from .spectrogram_augmentations import augment_freq_time_mask, augment_dropout, augment_pitch_and_tempo, augment_speed_up, augment_sparse_warp
from .audio import change_audio_types, read_frames_from_file, vad_split, pcm_to_np, DEFAULT_FORMAT, AUDIO_TYPE_NP
from .sample_collections import samples_from_files
from .audio import read_frames_from_file, vad_split, pcm_to_np, DEFAULT_FORMAT
from .sample_collections import samples_from_sources, augment_samples
from .helpers import remember_exception, MEGABYTE
@ -109,6 +109,8 @@ def to_sparse_tuple(sequence):
def create_dataset(sources,
batch_size,
repetitions=1,
augmentation_specs=None,
enable_cache=False,
cache_path=None,
train_phase=False,
@ -116,13 +118,16 @@ def create_dataset(sources,
process_ahead=None,
buffering=1 * MEGABYTE):
def generate_values():
samples = samples_from_files(sources, buffering=buffering, labeled=True)
for sample in change_audio_types(samples,
AUDIO_TYPE_NP,
process_ahead=2 * batch_size if process_ahead is None else process_ahead):
samples = samples_from_sources(sources, buffering=buffering, labeled=True)
samples = augment_samples(samples,
repetitions=repetitions,
augmentation_specs=augmentation_specs,
buffering=buffering,
process_ahead=2 * batch_size if process_ahead is None else process_ahead)
for sample in samples:
transcript = text_to_char_array(sample.transcript, Config.alphabet, context=sample.sample_id)
transcript = to_sparse_tuple(transcript)
yield sample.sample_id, sample.audio, sample.audio_format[0], transcript
yield sample.sample_id, sample.audio, sample.audio_format.rate, transcript
# Batching a dataset of 2D SparseTensors creates 3D batches, which fail
# when passed to tf.nn.ctc_loss, so we reshape them to remove the extra
@ -167,7 +172,7 @@ def split_audio_file(audio_path,
yield time_start, time_end, samples
def to_mfccs(time_start, time_end, samples):
features, features_len = samples_to_mfccs(samples, audio_format[0])
features, features_len = samples_to_mfccs(samples, audio_format.rate)
return time_start, time_end, features, features_len
def create_batch_set(bs, criteria):

3
training/deepspeech_training/util/flags.py
View File

@ -26,6 +26,9 @@ def create_flags():
# Data Augmentation
# ================
f.DEFINE_multi_string('augment', None, 'specifies an augmentation of the training samples. Format is "--augment operation[param1=value1, ...]"')
f.DEFINE_integer('augmentations_per_epoch', 1, 'how often the train set should be repeated and re-augmented per epoch')
f.DEFINE_float('data_aug_features_additive', 0, 'std of the Gaussian additive noise')
f.DEFINE_float('data_aug_features_multiplicative', 0, 'std of normal distribution around 1 for multiplicative noise')

50
training/deepspeech_training/util/helpers.py
View File

@ -3,8 +3,10 @@ import sys
import time
import heapq
import semver
import random
from multiprocessing import Pool
from collections import namedtuple
KILO = 1024
KILOBYTE = 1 * KILO
@ -13,6 +15,8 @@ GIGABYTE = KILO * MEGABYTE
TERABYTE = KILO * GIGABYTE
SIZE_PREFIX_LOOKUP = {'k': KILOBYTE, 'm': MEGABYTE, 'g': GIGABYTE, 't': TERABYTE}
ValueRange = namedtuple('ValueRange', 'start end r')
def parse_file_size(file_size):
file_size = file_size.lower().strip()
@ -79,11 +83,11 @@ class LimitingPool:
"""Limits unbound ahead-processing of multiprocessing.Pool's imap method
before items get consumed by the iteration caller.
This prevents OOM issues in situations where items represent larger memory allocations."""
def __init__(self, processes=None, process_ahead=None, sleeping_for=0.1):
def __init__(self, processes=None, initializer=None, initargs=None, process_ahead=None, sleeping_for=0.1):
self.process_ahead = os.cpu_count() if process_ahead is None else process_ahead
self.sleeping_for = sleeping_for
self.processed = 0
self.pool = Pool(processes=processes)
self.pool = Pool(processes=processes, initializer=initializer, initargs=initargs)
def __enter__(self):
return self
@ -100,6 +104,9 @@ class LimitingPool:
self.processed -= 1
yield obj
def terminate(self):
self.pool.terminate()
def __exit__(self, exc_type, exc_value, traceback):
self.pool.close()
@ -128,3 +135,42 @@ def remember_exception(iterable, exception_box=None):
except Exception as ex: # pylint: disable = broad-except
exception_box.exception = ex
return iterable if exception_box is None else do_iterate
def get_value_range(value, target_type):
if isinstance(value, str):
r = target_type(0)
parts = value.split('~')
if len(parts) == 2:
value = parts[0]
r = target_type(parts[1])
elif len(parts) > 2:
raise ValueError('Cannot parse value range')
parts = value.split(':')
if len(parts) == 1:
parts.append(parts[0])
elif len(parts) > 2:
raise ValueError('Cannot parse value range')
return ValueRange(target_type(parts[0]), target_type(parts[1]), r)
if isinstance(value, tuple):
if len(value) == 2:
return ValueRange(target_type(value[0]), target_type(value[1]), 0)
if len(value) == 3:
return ValueRange(target_type(value[0]), target_type(value[1]), target_type(value[2]))
raise ValueError('Cannot convert to ValueRange: Wrong tuple size')
return ValueRange(target_type(value), target_type(value), 0)
def int_range(value):
return get_value_range(value, int)
def float_range(value):
return get_value_range(value, float)
def pick_value_from_range(value_range, clock=None):
clock = random.random() if clock is None else max(0.0, min(1.0, float(clock)))
value = value_range.start + clock * (value_range.end - value_range.start)
value = random.uniform(value - value_range.r, value + value_range.r)
return round(value) if isinstance(value_range.start, int) else value

230
training/deepspeech_training/util/sample_collections.py
View File

@ -2,11 +2,14 @@
import os
import csv
import json
import random
from pathlib import Path
from functools import partial
from .helpers import MEGABYTE, GIGABYTE, Interleaved
from .audio import Sample, DEFAULT_FORMAT, AUDIO_TYPE_WAV, AUDIO_TYPE_OPUS, SERIALIZABLE_AUDIO_TYPES
from .signal_augmentations import parse_augmentation
from .helpers import MEGABYTE, GIGABYTE, Interleaved, LimitingPool
from .audio import Sample, DEFAULT_FORMAT, AUDIO_TYPE_OPUS, AUDIO_TYPE_NP, SERIALIZABLE_AUDIO_TYPES, get_audio_type_from_extension
BIG_ENDIAN = 'big'
INT_SIZE = 4
@ -47,9 +50,42 @@ class LabeledSample(Sample):
self.transcript = transcript
def load_sample(filename, label=None):
"""
Loads audio-file as a (labeled or unlabeled) sample
Parameters
----------
filename : str
Filename of the audio-file to load as sample
label : str
Label (transcript) of the sample.
If None: return util.audio.Sample instance
Otherwise: return util.sample_collections.LabeledSample instance
Returns
-------
util.audio.Sample instance if label is None, else util.sample_collections.LabeledSample instance
"""
ext = os.path.splitext(filename)[1].lower()
audio_type = get_audio_type_from_extension(ext)
if audio_type is None:
raise ValueError('Unknown audio type extension "{}"'.format(ext))
with open(filename, 'rb') as audio_file:
if label is None:
return Sample(audio_type, audio_file.read(), sample_id=filename)
return LabeledSample(audio_type, audio_file.read(), label, sample_id=filename)
class DirectSDBWriter:
"""Sample collection writer for creating a Sample DB (SDB) file"""
def __init__(self, sdb_filename, buffering=BUFFER_SIZE, audio_type=AUDIO_TYPE_OPUS, id_prefix=None, labeled=True):
def __init__(self,
sdb_filename,
buffering=BUFFER_SIZE,
audio_type=AUDIO_TYPE_OPUS,
bitrate=None,
id_prefix=None,
labeled=True):
"""
Parameters
----------
@ -59,6 +95,8 @@ class DirectSDBWriter:
Write-buffer size to use while writing the SDB file
audio_type : str
See util.audio.Sample.__init__ .
bitrate : int
Bitrate for sample-compression in case of lossy audio_type (e.g. AUDIO_TYPE_OPUS)
id_prefix : str
Prefix for IDs of written samples - defaults to sdb_filename
labeled : bool or None
@ -71,6 +109,7 @@ class DirectSDBWriter:
if audio_type not in SERIALIZABLE_AUDIO_TYPES:
raise ValueError('Audio type "{}" not supported'.format(audio_type))
self.audio_type = audio_type
self.bitrate = bitrate
self.sdb_file = open(sdb_filename, 'wb', buffering=buffering)
self.offsets = []
self.num_samples = 0
@ -100,7 +139,7 @@ class DirectSDBWriter:
def add(self, sample):
def to_bytes(n):
return n.to_bytes(INT_SIZE, BIG_ENDIAN)
sample.change_audio_type(self.audio_type)
sample.change_audio_type(self.audio_type, bitrate=self.bitrate)
opus = sample.audio.getbuffer()
opus_len = to_bytes(len(opus))
if self.labeled:
@ -260,7 +299,31 @@ class SDB: # pylint: disable=too-many-instance-attributes
self.close()
class CSV:
class SampleList:
"""Sample collection base class with samples loaded from a list of in-memory paths."""
def __init__(self, samples, labeled=True):
"""
Parameters
----------
samples : iterable of tuples of the form (sample_filename, filesize [, transcript])
File-size is used for ordering the samples; transcript has to be provided if labeled=True
labeled : bool or None
If True: Reads LabeledSample instances.
If False: Ignores transcripts (if available) and reads (unlabeled) util.audio.Sample instances.
"""
self.labeled = labeled
self.samples = list(samples)
self.samples.sort(key=lambda r: r[1])
def __getitem__(self, i):
sample_spec = self.samples[i]
return load_sample(sample_spec[0], label=sample_spec[2] if self.labeled else None)
def __len__(self):
return len(self.samples)
class CSV(SampleList):
"""Sample collection reader for reading a DeepSpeech CSV file
Automatically orders samples by CSV column wav_filesize (if available)."""
def __init__(self, csv_filename, labeled=None):
@ -275,16 +338,14 @@ class CSV:
If None: Automatically determines if CSV file has a transcript column
(reading util.sample_collections.LabeledSample instances) or not (reading util.audio.Sample instances).
"""
self.csv_filename = csv_filename
self.labeled = labeled
self.rows = []
rows = []
csv_dir = Path(csv_filename).parent
with open(csv_filename, 'r', encoding='utf8') as csv_file:
reader = csv.DictReader(csv_file)
if 'transcript' in reader.fieldnames:
if self.labeled is None:
self.labeled = True
elif self.labeled:
if labeled is None:
labeled = True
elif labeled:
raise RuntimeError('No transcript data (missing CSV column)')
for row in reader:
wav_filename = Path(row['wav_filename'])
@ -292,36 +353,20 @@ class CSV:
wav_filename = csv_dir / wav_filename
wav_filename = str(wav_filename)
wav_filesize = int(row['wav_filesize']) if 'wav_filesize' in row else 0
if self.labeled:
self.rows.append((wav_filename, wav_filesize, row['transcript']))
if labeled:
rows.append((wav_filename, wav_filesize, row['transcript']))
else:
self.rows.append((wav_filename, wav_filesize))
self.rows.sort(key=lambda r: r[1])
def __getitem__(self, i):
row = self.rows[i]
wav_filename = row[0]
with open(wav_filename, 'rb') as wav_file:
if self.labeled:
return LabeledSample(AUDIO_TYPE_WAV, wav_file.read(), row[2], sample_id=wav_filename)
return Sample(AUDIO_TYPE_WAV, wav_file.read(), sample_id=wav_filename)
def __iter__(self):
for i in range(len(self.rows)):
yield self[i]
def __len__(self):
return len(self.rows)
rows.append((wav_filename, wav_filesize))
super(CSV, self).__init__(rows, labeled=labeled)
def samples_from_file(filename, buffering=BUFFER_SIZE, labeled=None):
def samples_from_source(sample_source, buffering=BUFFER_SIZE, labeled=None):
"""
Returns an iterable of util.sample_collections.LabeledSample or util.audio.Sample instances
loaded from a sample source file.
Loads samples from a sample source file.
Parameters
----------
filename : str
sample_source : str
Path to the sample source file (SDB or CSV)
buffering : int
Read-buffer size to use while reading files
@ -330,23 +375,27 @@ def samples_from_file(filename, buffering=BUFFER_SIZE, labeled=None):
If False: Ignores transcripts (if available) and reads (unlabeled) util.audio.Sample instances.
If None: Automatically determines if source provides transcripts
(reading util.sample_collections.LabeledSample instances) or not (reading util.audio.Sample instances).
Returns
-------
iterable of util.sample_collections.LabeledSample or util.audio.Sample instances supporting len.
"""
ext = os.path.splitext(filename)[1].lower()
ext = os.path.splitext(sample_source)[1].lower()
if ext == '.sdb':
return SDB(filename, buffering=buffering, labeled=labeled)
return SDB(sample_source, buffering=buffering, labeled=labeled)
if ext == '.csv':
return CSV(filename, labeled=labeled)
return CSV(sample_source, labeled=labeled)
raise ValueError('Unknown file type: "{}"'.format(ext))
def samples_from_files(filenames, buffering=BUFFER_SIZE, labeled=None):
def samples_from_sources(sample_sources, buffering=BUFFER_SIZE, labeled=None):
"""
Returns an iterable of util.sample_collections.LabeledSample or util.audio.Sample instances
loaded from a collection of sample source files.
Loads and combines samples from a list of source files. Sources are combined in an interleaving way to
keep default sample order from shortest to longest.
Parameters
----------
filenames : list of str
sample_sources : list of str
Paths to sample source files (SDBs or CSVs)
buffering : int
Read-buffer size to use while reading files
@ -355,11 +404,100 @@ def samples_from_files(filenames, buffering=BUFFER_SIZE, labeled=None):
If False: Ignores transcripts (if available) and always reads (unlabeled) util.audio.Sample instances.
If None: Reads util.sample_collections.LabeledSample instances from sources with transcripts and
util.audio.Sample instances from sources with no transcripts.
Returns
-------
iterable of util.sample_collections.LabeledSample (labeled=True) or util.audio.Sample (labeled=False) supporting len
"""
filenames = list(filenames)
if len(filenames) == 0:
sample_sources = list(sample_sources)
if len(sample_sources) == 0:
raise ValueError('No files')
if len(filenames) == 1:
return samples_from_file(filenames[0], buffering=buffering, labeled=labeled)
cols = list(map(partial(samples_from_file, buffering=buffering, labeled=labeled), filenames))
if len(sample_sources) == 1:
return samples_from_source(sample_sources[0], buffering=buffering, labeled=labeled)
cols = list(map(partial(samples_from_source, buffering=buffering, labeled=labeled), sample_sources))
return Interleaved(*cols, key=lambda s: s.duration)
class PreparationContext:
def __init__(self, target_audio_type, augmentations):
self.target_audio_type = target_audio_type
self.augmentations = augmentations
AUGMENTATION_CONTEXT = None
def _init_augmentation_worker(preparation_context):
global AUGMENTATION_CONTEXT # pylint: disable=global-statement
AUGMENTATION_CONTEXT = preparation_context
def _augment_sample(timed_sample, context=None):
context = AUGMENTATION_CONTEXT if context is None else context
sample, clock = timed_sample
for augmentation in context.augmentations:
if random.random() < augmentation.probability:
augmentation.apply(sample, clock)
sample.change_audio_type(new_audio_type=context.target_audio_type)
return sample
def augment_samples(samples,
audio_type=AUDIO_TYPE_NP,
augmentation_specs=None,
buffering=BUFFER_SIZE,
process_ahead=None,
repetitions=1,
fixed_clock=None):
"""
Prepares samples for being used during training.
This includes parallel and buffered application of augmentations and a conversion to a specified audio-type.
Parameters
----------
samples : Sample enumeration
Typically produced by samples_from_sources.
audio_type : str
Target audio-type to convert samples to. See util.audio.Sample.__init__ .
augmentation_specs : list of str
Augmentation specifications like ["reverb[delay=20.0,decay=-20]", "volume"]. See TRAINING.rst.
buffering : int
Read-buffer size to use while reading files.
process_ahead : int
Number of samples to pre-process ahead of time.
repetitions : int
How often the input sample enumeration should get repeated for being re-augmented.
fixed_clock : float
Sets the internal clock to a value between 0.0 (beginning of epoch) and 1.0 (end of epoch).
Setting this to a number is used for simulating augmentations at a certain epoch-time.
If kept at None (default), the internal clock will run regularly from 0.0 to 1.0,
hence preparing them for training.
Returns
-------
iterable of util.sample_collections.LabeledSample or util.audio.Sample
"""
def timed_samples():
for repetition in range(repetitions):
for sample_index, sample in enumerate(samples):
if fixed_clock is None:
yield sample, (repetition * len(samples) + sample_index) / (repetitions * len(samples))
else:
yield sample, fixed_clock
augmentations = [] if augmentation_specs is None else list(map(parse_augmentation, augmentation_specs))
try:
for augmentation in augmentations:
augmentation.start(buffering=buffering)
context = PreparationContext(audio_type, augmentations)
if process_ahead == 0:
for timed_sample in timed_samples():
yield _augment_sample(timed_sample, context=context)
else:
with LimitingPool(process_ahead=process_ahead,
initializer=_init_augmentation_worker,
initargs=(context,)) as pool:
yield from pool.imap(_augment_sample, timed_samples())
finally:
for augmentation in augmentations:
augmentation.stop()

222
training/deepspeech_training/util/signal_augmentations.py Normal file
View File

@ -0,0 +1,222 @@
import os
import re
import math
import random
import numpy as np
from multiprocessing import Queue, Process
from .audio import gain_db_to_ratio, max_dbfs, normalize_audio, AUDIO_TYPE_NP, AUDIO_TYPE_PCM, AUDIO_TYPE_OPUS
from .helpers import int_range, float_range, pick_value_from_range, MEGABYTE
SPEC_PARSER = re.compile(r'^(?P<cls>[a-z]+)(\[(?P<params>.*)\])?$')
BUFFER_SIZE = 1 * MEGABYTE
class Augmentation:
def __init__(self, p=1.0):
self.probability = float(p)
def start(self, buffering=BUFFER_SIZE):
pass
def apply(self, sample, clock):
raise NotImplementedError
def stop(self):
pass
def _enqueue_overlay_samples(sample_source, queue, buffering=BUFFER_SIZE):
"""
As the central distribution point for overlay samples this function is supposed to run in one process only.
This ensures that samples are not used twice if not required.
It loads the (raw and still compressed) data and provides it to the actual augmentation workers.
These are then doing decompression, potential conversion and overlaying in parallel.
"""
# preventing cyclic import problems
from .sample_collections import samples_from_source # pylint: disable=import-outside-toplevel
samples = samples_from_source(sample_source, buffering=buffering, labeled=False)
while True:
for sample in samples:
queue.put(sample)
class Overlay(Augmentation):
"""See "Overlay augmentation" in TRAINING.rst"""
def __init__(self, source, p=1.0, snr=3.0, layers=1):
super(Overlay, self).__init__(p)
self.source = source
self.snr = float_range(snr)
self.layers = int_range(layers)
self.queue = Queue(max(1, math.floor(self.probability * self.layers[1] * os.cpu_count())))
self.current_sample = None
self.enqueue_process = None
def start(self, buffering=BUFFER_SIZE):
self.enqueue_process = Process(target=_enqueue_overlay_samples,
args=(self.source, self.queue),
kwargs={'buffering': buffering})
self.enqueue_process.start()
def apply(self, sample, clock):
sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
n_layers = pick_value_from_range(self.layers, clock=clock)
audio = sample.audio
overlay_data = np.zeros_like(audio)
for _ in range(n_layers):
overlay_offset = 0
while overlay_offset < len(audio):
if self.current_sample is None:
next_overlay_sample = self.queue.get()
next_overlay_sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
self.current_sample = next_overlay_sample.audio
n_required = len(audio) - overlay_offset
n_current = len(self.current_sample)
if n_required >= n_current: # take it completely
overlay_data[overlay_offset:overlay_offset + n_current] += self.current_sample
overlay_offset += n_current
self.current_sample = None
else: # take required slice from head and keep tail for next layer or sample
overlay_data[overlay_offset:overlay_offset + n_required] += self.current_sample[0:n_required]
overlay_offset += n_required
self.current_sample = self.current_sample[n_required:]
snr_db = pick_value_from_range(self.snr, clock=clock)
orig_dbfs = max_dbfs(audio)
overlay_gain = orig_dbfs - max_dbfs(overlay_data) - snr_db
audio += overlay_data * gain_db_to_ratio(overlay_gain)
sample.audio = normalize_audio(audio, dbfs=orig_dbfs)
def stop(self):
if self.enqueue_process is not None:
self.enqueue_process.terminate()
class Reverb(Augmentation):
"""See "Reverb augmentation" in TRAINING.rst"""
def __init__(self, p=1.0, delay=20.0, decay=10.0):
super(Reverb, self).__init__(p)
self.delay = float_range(delay)
self.decay = float_range(decay)
def apply(self, sample, clock):
sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
audio = np.array(sample.audio, dtype=np.float64)
orig_dbfs = max_dbfs(audio)
delay = pick_value_from_range(self.delay, clock=clock)
decay = pick_value_from_range(self.decay, clock=clock)
decay = gain_db_to_ratio(-decay)
result = np.copy(audio)
primes = [17, 19, 23, 29, 31]
for delay_prime in primes: # primes to minimize comb filter interference
layer = np.copy(audio)
n_delay = math.floor(delay * (delay_prime / primes[0]) * sample.audio_format.rate / 1000.0)
n_delay = max(16, n_delay) # 16 samples minimum to avoid performance trap and risk of division by zero
for w_index in range(0, math.floor(len(audio) / n_delay)):
w1 = w_index * n_delay
w2 = (w_index + 1) * n_delay
width = min(len(audio) - w2, n_delay) # last window could be smaller
layer[w2:w2 + width] += decay * layer[w1:w1 + width]
result += layer
audio = normalize_audio(result, dbfs=orig_dbfs)
sample.audio = np.array(audio, dtype=np.float32)
class Resample(Augmentation):
"""See "Resample augmentation" in TRAINING.rst"""
def __init__(self, p=1.0, rate=8000):
super(Resample, self).__init__(p)
self.rate = int_range(rate)
def apply(self, sample, clock):
# late binding librosa and its dependencies
from librosa.core import resample # pylint: disable=import-outside-toplevel
sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
rate = pick_value_from_range(self.rate, clock=clock)
audio = sample.audio
orig_len = len(audio)
audio = np.swapaxes(audio, 0, 1)
audio = resample(audio, sample.audio_format.rate, rate)
audio = resample(audio, rate, sample.audio_format.rate)
audio = np.swapaxes(audio, 0, 1)[0:orig_len]
sample.audio = audio
class Codec(Augmentation):
"""See "Codec augmentation" in TRAINING.rst"""
def __init__(self, p=1.0, bitrate=3200):
super(Codec, self).__init__(p)
self.bitrate = int_range(bitrate)
def apply(self, sample, clock):
bitrate = pick_value_from_range(self.bitrate, clock=clock)
sample.change_audio_type(new_audio_type=AUDIO_TYPE_PCM) # decoding to ensure it has to get encoded again
sample.change_audio_type(new_audio_type=AUDIO_TYPE_OPUS, bitrate=bitrate) # will get decoded again downstream
class Gaps(Augmentation):
"""See "Gaps augmentation" in TRAINING.rst"""
def __init__(self, p=1.0, n=1, size=50.0):
super(Gaps, self).__init__(p)
self.n_gaps = int_range(n)
self.size = float_range(size)
def apply(self, sample, clock):
sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
audio = sample.audio
n_gaps = pick_value_from_range(self.n_gaps, clock=clock)
for _ in range(n_gaps):
size = pick_value_from_range(self.size, clock=clock)
size = int(size * sample.audio_format.rate / 1000.0)
size = min(size, len(audio) // 10) # a gap should never exceed 10 percent of the audio
offset = random.randint(0, max(0, len(audio) - size - 1))
audio[offset:offset + size] = 0
sample.audio = audio
class Volume(Augmentation):
"""See "Volume augmentation" in TRAINING.rst"""
def __init__(self, p=1.0, dbfs=3.0103):
super(Volume, self).__init__(p)
self.target_dbfs = float_range(dbfs)
def apply(self, sample, clock):
sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP)
target_dbfs = pick_value_from_range(self.target_dbfs, clock=clock)
sample.audio = normalize_audio(sample.audio, dbfs=target_dbfs)
def parse_augmentation(augmentation_spec):
"""
Parses an augmentation specification.
Parameters
----------
augmentation_spec : str
Augmentation specification like "reverb[delay=20.0,decay=-20]".
Returns
-------
Instance of an augmentation class from util.signal_augmentations.*.
"""
match = SPEC_PARSER.match(augmentation_spec)
if not match:
raise ValueError('Augmentation specification has wrong format')
cls_name = match.group('cls')
cls_name = cls_name[0].upper() + cls_name[1:]
augmentation_cls = globals()[cls_name] if cls_name in globals() else None
if not issubclass(augmentation_cls, Augmentation) or augmentation_cls == Augmentation:
raise ValueError('Unknown augmentation: {}'.format(cls_name))
parameters = match.group('params')
parameters = [] if parameters is None else parameters.split(',')
args = []
kwargs = {}
for parameter in parameters:
pair = tuple(list(map(str.strip, (parameter.split('=')))))
if len(pair) == 1:
args.append(pair)
elif len(pair) == 2:
kwargs[pair[0]] = pair[1]
else:
raise ValueError('Unable to parse augmentation value assignment')
return augmentation_cls(*args, **kwargs)