mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into update-tf-master

Browse Source
This commit is contained in:
Alexandre Lissy 2018-11-06 08:02:10 +01:00
commit 706b54f264
24 changed files with 116 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.install
View File

@ -3,7 +3,7 @@
virtualenv -p python3 ../tmp/venv
source ../tmp/venv/bin/activate
pip install -r <(grep -v tensorflow requirements.txt)
pip install tf-nightly==1.12.0.dev20181004
pip install tensorflow-gpu==1.12.0rc2
python3 util/taskcluster.py --arch cuda --target ../tmp/native_client

87
DeepSpeech.py
View File

@ -18,10 +18,12 @@ import time
import traceback
import inspect
import progressbar
import tempfile
from functools import partial
from six.moves import zip, range, filter, urllib, BaseHTTPServer
from tensorflow.python.tools import freeze_graph
from tensorflow.contrib.lite.python import tflite_convert
from threading import Thread, Lock
from util.audio import audiofile_to_input_vector
from util.feeding import DataSet, ModelFeeder
@ -1831,9 +1833,8 @@ def create_inference_graph(batch_size=1, n_steps=16, use_new_decoder=False, tfli
return (
{
'input': input_tensor,
'input_lengths': seq_length,
'new_state_c': new_state_c,
'new_state_h': new_state_h,
'previous_state_c': previous_state_c,
'previous_state_h': previous_state_h,
},
{
'outputs': logits,
@ -1849,11 +1850,17 @@ def export():
'''
log_info('Exporting the model...')
with tf.device('/cpu:0'):
from tensorflow.python.framework.ops import Tensor, Operation
tf.reset_default_graph()
session = tf.Session(config=session_config)
inputs, outputs = create_inference_graph(batch_size=1, n_steps=FLAGS.n_steps, tflite=FLAGS.export_tflite)
input_names = ",".join(tensor.op.name for tensor in inputs.values())
output_names_tensors = [ tensor.op.name for tensor in outputs.values() if isinstance(tensor, Tensor) ]
output_names_ops = [ tensor.name for tensor in outputs.values() if isinstance(tensor, Operation) ]
output_names = ",".join(output_names_tensors + output_names_ops)
input_shapes = ":".join(",".join(map(str, tensor.shape)) for tensor in inputs.values())
if not FLAGS.export_tflite:
mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')}
@ -1872,11 +1879,7 @@ def export():
checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
checkpoint_path = checkpoint.model_checkpoint_path
if not FLAGS.export_tflite:
output_filename = 'output_graph.pb'
else:
output_filename = 'output_graph.fb'
output_filename = 'output_graph.pb'
if FLAGS.remove_export:
if os.path.isdir(FLAGS.export_dir):
log_info('Removing old export')
@ -1887,31 +1890,61 @@ def export():
if not os.path.isdir(FLAGS.export_dir):
os.makedirs(FLAGS.export_dir)
if not FLAGS.export_tflite:
output_node_names = 'logits,initialize_state'
variables_blacklist = 'previous_state_c,previous_state_h'
else:
output_node_names = 'logits,new_state_c,new_state_h'
variables_blacklist = ''
def do_graph_freeze(output_file=None, output_node_names=None, variables_blacklist=None):
freeze_graph.freeze_graph_with_def_protos(
input_graph_def=session.graph_def,
input_saver_def=saver.as_saver_def(),
input_checkpoint=checkpoint_path,
output_node_names=output_node_names,
restore_op_name=None,
filename_tensor_name=None,
output_graph=output_file,
clear_devices=False,
variable_names_blacklist=variables_blacklist,
initializer_nodes='')
# Freeze graph
freeze_graph.freeze_graph_with_def_protos(
input_graph_def=session.graph_def,
input_saver_def=saver.as_saver_def(),
input_checkpoint=checkpoint_path,
output_node_names=output_node_names,
restore_op_name=None,
filename_tensor_name=None,
output_graph=output_graph_path,
clear_devices=False,
variable_names_blacklist=variables_blacklist,
initializer_nodes='')
if not FLAGS.export_tflite:
do_graph_freeze(output_file=output_graph_path, output_node_names=output_names, variables_blacklist='previous_state_c,previous_state_h')
else:
temp_fd, temp_freeze = tempfile.mkstemp(dir=FLAGS.export_dir)
os.close(temp_fd)
do_graph_freeze(output_file=temp_freeze, output_node_names=output_names, variables_blacklist='')
output_tflite_path = os.path.join(FLAGS.export_dir, output_filename.replace('.pb', '.tflite'))
class TFLiteFlags():
def __init__(self):
self.graph_def_file = temp_freeze
self.inference_type = 'FLOAT'
self.input_arrays = input_names
self.input_shapes = input_shapes
self.output_arrays = output_names
self.output_file = output_tflite_path
self.output_format = 'TFLITE'
default_empty = [
'inference_input_type',
'mean_values',
'default_ranges_min', 'default_ranges_max',
'drop_control_dependency',
'reorder_across_fake_quant',
'change_concat_input_ranges',
'allow_custom_ops',
'converter_mode',
'post_training_quantize',
'dump_graphviz_dir',
'dump_graphviz_video'
]
for e in default_empty:
self.__dict__[e] = None
flags = TFLiteFlags()
tflite_convert._convert_model(flags)
os.unlink(temp_freeze)
log_info('Exported model for TF Lite engine as {}'.format(os.path.basename(output_tflite_path)))
log_info('Models exported at %s' % (FLAGS.export_dir))
except RuntimeError as e:
log_error(str(e))
def do_single_file_inference(input_file_path):
with tf.Session(config=session_config) as session:
inputs, outputs = create_inference_graph(batch_size=1, use_new_decoder=True)

4
Dockerfile
View File

@ -62,7 +62,7 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \
# Clone TensoFlow from Mozilla repo
RUN git clone https://github.com/mozilla/tensorflow/
WORKDIR /tensorflow
RUN git checkout r1.11
RUN git checkout r1.12
# GPU Environment Setup
@ -190,7 +190,7 @@ RUN cp /tensorflow/bazel-bin/native_client/libctc_decoder_with_kenlm.so /DeepSpe
# Install TensorFlow
WORKDIR /DeepSpeech/
RUN pip install tf-nightly==1.12.0.dev20181004
RUN pip install tensorflow-gpu==1.12.0rc2
# Make DeepSpeech and install Python bindings

9
README.md
View File

@ -45,6 +45,7 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech
- [Training a model](#training-a-model)
- [Checkpointing](#checkpointing)
- [Exporting a model for inference](#exporting-a-model-for-inference)
- [Exporting a model for TFLite](#exporting-a-model-for-tflite)
- [Distributed computing across more than one machine](#distributed-training-across-more-than-one-machine)
- [Continuing training from a release model](#continuing-training-from-a-release-model)
- [Code documentation](#code-documentation)
@ -226,7 +227,7 @@ If you have a capable (Nvidia, at least 8GB of VRAM) GPU, it is highly recommend
```bash
pip3 uninstall tensorflow
pip3 install 'tensorflow-gpu==1.11.0'
pip3 install 'tensorflow-gpu==1.12.0rc2'
```
### Common Voice training data
@ -317,12 +318,16 @@ Be aware however that checkpoints are only valid for the same model geometry the
If the `--export_dir` parameter is provided, a model will have been exported to this directory during training.
Refer to the corresponding [README.md](native_client/README.md) for information on building and running a client that can use the exported model.
### Exporting a model for TFLite
If you want to experiment with the TF Lite engine, you need to export a model that is compatible with it, then use the `--export_tflite` flag. If you already have a trained model, you can re-export it for TFLite by running `DeepSpeech.py` again and specifying the same `checkpoint_dir` that you used for training, as well as passing `--notrain --notest --export_tflite --export_dir /model/export/destination`.
### Making a mmap-able model for inference
The `output_graph.pb` model file generated in the above step will be loaded in memory to be dealt with when running inference.
This will result in extra loading time and memory consumption. One way to avoid this is to directly read data from the disk.
TensorFlow has tooling to achieve this: it requires building the target `//tensorflow/contrib/util:convert_graphdef_memmapped_format` (binaries are produced by our TaskCluster for some systems including Linux/amd64 and macOS/amd64), use `util/taskcluster.py` tool to download, specifying `tensorflow` as a source.
TensorFlow has tooling to achieve this: it requires building the target `//tensorflow/contrib/util:convert_graphdef_memmapped_format` (binaries are produced by our TaskCluster for some systems including Linux/amd64 and macOS/amd64), use `util/taskcluster.py` tool to download, specifying `tensorflow` as a source and `convert_graphdef_memmapped_format` as artifact.
Producing a mmap-able model is as simple as:
```
$ convert_graphdef_memmapped_format --in_graph=output_graph.pb --out_graph=output_graph.pbmm

2
VERSION
View File

@ -1 +1 @@
0.3.0
0.4.0-alpha.0

2
native_client/README.md
View File

@ -52,7 +52,7 @@ Check the [main README](../README.md) for more details.
If you'd like to build the binaries yourself, you'll need the following pre-requisites downloaded/installed:
* [TensorFlow requirements](https://www.tensorflow.org/install/install_sources)
* [TensorFlow `r1.11` sources](https://github.com/mozilla/tensorflow/tree/r1.11)
* [TensorFlow `r1.12` sources](https://github.com/mozilla/tensorflow/tree/r1.12)
* [libsox](https://sourceforge.net/projects/sox/)
It is required to use our fork of TensorFlow since it includes fixes for common problems encountered when building the native client files.

2
requirements.txt
View File

@ -1,7 +1,7 @@
pandas
progressbar2
python-utils
tf-nightly==1.12.0.dev20181004
tensorflow == 1.12.0rc2
numpy
matplotlib
scipy

1
taskcluster/.build.yml
View File

@ -21,3 +21,4 @@ build:
args:
tests_cmdline: ''
convert_graphdef: ''
benchmark_model_bin: ''

1
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -7,7 +7,6 @@ build:
- "notify.irc-channel.${notifications.irc}.on-exception"
- "notify.irc-channel.${notifications.irc}.on-failed"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.osx/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

1
taskcluster/darwin-opt-base.tyml
View File

@ -39,7 +39,6 @@ payload:
training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") }
in:
TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph}
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
# There is no VM yet running tasks on OSX

1
taskcluster/linux-amd64-cpu-opt.yml
View File

@ -15,7 +15,6 @@ build:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

1
taskcluster/linux-amd64-ctc-opt.yml
View File

@ -5,7 +5,6 @@ build:
- "pull_request.reopened"
template_file: linux-opt-base.tyml
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/summarize_graph"
scripts:
build: 'taskcluster/decoder-build.sh'
package: 'taskcluster/decoder-package.sh'

1
taskcluster/linux-amd64-cuda-opt.yml
View File

@ -13,7 +13,6 @@ build:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cuda/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cuda/artifacts/public/summarize_graph"
maxRunTime: 14400
scripts:
build: "taskcluster/cuda-build.sh"

1
taskcluster/linux-arm64-cpu-opt.yml
View File

@ -5,7 +5,6 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
- "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.arm64/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

1
taskcluster/linux-opt-base.tyml
View File

@ -36,7 +36,6 @@ then:
training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") }
in:
TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph}
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
command:

1
taskcluster/linux-rpi3-cpu-opt.yml
View File

@ -5,7 +5,6 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
- "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

1
taskcluster/node-package.yml
View File

@ -17,7 +17,6 @@ build:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.5a9434129637a0e4cc4e5a02f21f4a505cda5ac5.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/node-build.sh"
package: "taskcluster/node-package.sh"

1
taskcluster/test-armbian-opt-base.tyml
View File

@ -35,7 +35,6 @@ then:
linux_arm64_build: { $eval: as_slugid("linux-arm64-cpu-opt") }
node_package: { $eval: as_slugid("node-package") }
in:
CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef}
DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_arm64_build}/artifacts/public
DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb

1
taskcluster/test-linux-opt-base.tyml
View File

@ -37,6 +37,7 @@ then:
node_package: { $eval: as_slugid("node-package") }
in:
CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef}
BENCHMARK_MODEL_BIN: ${build.benchmark_model_bin}
DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public
DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz

10
taskcluster/test-lite_benchmark_model-linux-amd64-opt.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "test-training_upstream-linux-amd64-py27mu-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-lite_benchmark_model-ds-tests.sh"
benchmark_model_bin: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.12.bea86c1e884730cf7f8615eb24d31872c198c766.cpu/artifacts/public/lite_benchmark_model"
metadata:
name: "DeepSpeech Linux AMD64 CPU TF Lite benchmark_model"
description: "Testing DeepSpeech TF Lite benchmark_model for Linux/AMD64, CPU only, optimized version"

1
taskcluster/test-raspbian-opt-base.tyml
View File

@ -35,7 +35,6 @@ then:
linux_rpi3_build: { $eval: as_slugid("linux-rpi3-cpu-opt") }
node_package: { $eval: as_slugid("node-package") }
in:
CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef}
DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_rpi3_build}/artifacts/public
DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb

20
tc-lite_benchmark_model-ds-tests.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
download_benchmark_model "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
lite_benchmark_model \
--graph=${TASKCLUSTER_TMP_DIR}/ds/${model_name} \
--show_flops \
--input_layer=input_node,previous_state_c,previous_state_h \
--input_layer_type=float,float,float \
--input_layer_shape=1,16,19,26:1:1,494:1,494 \
--output_layer=logits,new_state_c,new_state_h

10
tc-tests-utils.sh
View File

@ -275,6 +275,16 @@ download_material()
ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav ${TASKCLUSTER_TMP_DIR}/alphabet.txt
}
download_benchmark_model()
{
target_dir=$1
mkdir -p ${target_dir} || true
wget -P "${target_dir}" "${model_source}"
wget -P "${target_dir}" "${BENCHMARK_MODEL_BIN}" && chmod +x ${target_dir}/*benchmark_model
}
install_pyenv()
{
if [ -z "${PYENV_ROOT}" ]; then

2
tc-train-tests.sh
View File

@ -66,7 +66,7 @@ pushd ${HOME}/DeepSpeech/ds/
popd
cp /tmp/train/output_graph.pb ${TASKCLUSTER_ARTIFACTS}
cp /tmp/train/output_graph.fb ${TASKCLUSTER_ARTIFACTS}
cp /tmp/train/output_graph.tflite ${TASKCLUSTER_ARTIFACTS}
if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then
convert_graphdef=$(basename "${CONVERT_GRAPHDEF_MEMMAPPED}")