mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1627 from lissyx/tf-master-20181004

Browse Source 
Tf master 20181004
This commit is contained in:
lissyx 2018-10-05 17:09:23 +02:00 committed by GitHub
commit 2857f10a1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 44 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.install
View File

@ -3,8 +3,8 @@
virtualenv -p python3 ../tmp/venv
source ../tmp/venv/bin/activate
pip install -r <(grep -v tensorflow requirements.txt)
pip install tensorflow-gpu==1.11.0
pip install tf-nightly==1.12.0.dev20181004
python3 util/taskcluster.py --arch gpu --target ../tmp/native_client
python3 util/taskcluster.py --arch cuda --target ../tmp/native_client
mkdir -p ../keep/summaries

3
DeepSpeech.py
View File

@ -1646,6 +1646,9 @@ def train(server=None):
update_progressbar.current_job_index += 1
# Initialize update_progressbar()'s child fields to safe values
update_progressbar.pbar = None
# The MonitoredTrainingSession takes care of session initialization,
# restoring from a checkpoint, saving to a checkpoint, and closing when done
# or an error occurs.

2
Dockerfile
View File

@ -190,7 +190,7 @@ RUN cp /tensorflow/bazel-bin/native_client/libctc_decoder_with_kenlm.so /DeepSpe
# Install TensorFlow
WORKDIR /DeepSpeech/
RUN pip install tensorflow-gpu==1.11.0
RUN pip install tf-nightly==1.12.0.dev20181004
# Make DeepSpeech and install Python bindings

2
bin/benchmark_nc.py
View File

@ -84,7 +84,7 @@ def get_arch_string():
nv_rc, nv_stdout, nv_stderr = exec_command('nvidia-smi')
nv_stdout = nv_stdout.lower().strip()
if 'NVIDIA-SMI' in nv_stdout:
return 'gpu'
return 'cuda'
else:
return 'cpu'

2
native_client/README.md
View File

@ -17,7 +17,7 @@ python3 util/taskcluster.py --branch "v0.2.0-alpha.6"
This will download and extract `native_client.tar.xz` which includes the deepspeech binary and associated libraries as well as the custom decoder OP. `taskcluster.py` will download binaries for the architecture of the host by default, but you can override that behavior with the `--arch` parameter. See the help info with `python util/taskcluster.py -h` for more details.
If you want the CUDA capable version of the binaries, use `--arch gpu`. Note that for now we don't publish CUDA-capable macOS binaries.
If you want the CUDA capable version of the binaries, use `--arch cuda`. Note that TensorFlow does not support CUDA on macOS anymore.
If you're looking to train a model, you now have a `libctc_decoder_with_kenlm.so` file that you can pass to the `--decoder_library_path` parameter of `DeepSpeech.py`.

2
requirements.txt
View File

@ -1,7 +1,7 @@
pandas
progressbar2
python-utils
tensorflow == 1.11.0
tf-nightly==1.12.0.dev20181004
numpy
matplotlib
scipy

4
taskcluster/cuda-build.sh
View File

@ -21,8 +21,8 @@ do_bazel_build
do_deepspeech_binary_build
do_deepspeech_python_build rename_to_gpu
do_deepspeech_python_build rename_to_cuda
do_deepspeech_nodejs_build rename_to_gpu
do_deepspeech_nodejs_build rename_to_cuda
$(dirname "$0")/decoder-build.sh

4
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -6,8 +6,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
- "notify.irc-channel.${notifications.irc}.on-exception"
- "notify.irc-channel.${notifications.irc}.on-failed"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.osx/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.osx/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

4
taskcluster/linux-amd64-cpu-opt.yml
View File

@ -14,8 +14,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

4
taskcluster/linux-amd64-ctc-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "pull_request.synchronize"
- "pull_request.reopened"
template_file: linux-opt-base.tyml
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph"
scripts:
build: 'taskcluster/decoder-build.sh'
package: 'taskcluster/decoder-package.sh'

10
taskcluster/linux-amd64-gpu-opt.yml → taskcluster/linux-amd64-cuda-opt.yml
View File

@ -1,9 +1,9 @@
build:
template_file: linux-opt-base.tyml
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"
- "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cuda"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cuda"
- "index.project.deepspeech.deepspeech.native_client.cuda.${event.head.sha}"
system_setup:
>
${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning}
@ -12,8 +12,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.gpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.gpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cuda/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cuda/artifacts/public/summarize_graph"
maxRunTime: 14400
scripts:
build: "taskcluster/cuda-build.sh"

4
taskcluster/linux-arm64-cpu-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
- "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.arm64/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.arm64/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/linux-rpi3-cpu-opt.yml
View File

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
- "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/node-package.yml
View File

@ -16,8 +16,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/node-build.sh"
package: "taskcluster/node-package.sh"

8
taskcluster/scriptworker-task.yml
View File

@ -5,7 +5,7 @@ build:
# - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org
- "darwin-amd64-cpu-opt"
- "linux-amd64-cpu-opt"
- "linux-amd64-gpu-opt"
- "linux-amd64-cuda-opt"
- "linux-rpi3-cpu-opt"
- "node-package"
allowed:
@ -18,18 +18,18 @@ build:
python:
- "darwin-amd64-cpu-opt"
- "linux-amd64-cpu-opt"
- "linux-amd64-gpu-opt"
- "linux-amd64-cuda-opt"
- "linux-rpi3-cpu-opt"
# - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org
javascript:
# GPU package
- "linux-amd64-gpu-opt"
- "linux-amd64-cuda-opt"
# CPU package with all archs
- "node-package"
cpp:
- "darwin-amd64-cpu-opt"
- "linux-amd64-cpu-opt"
- "linux-amd64-gpu-opt"
- "linux-amd64-cuda-opt"
- "linux-rpi3-cpu-opt"
- "linux-arm64-cpu-opt"
metadata:

2
taskcluster/test-armbian-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
PIP_DEFAULT_TIMEOUT: "60"
PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/"
EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9"
command:
- "/bin/bash"

2
taskcluster/test-darwin-opt-base.tyml
View File

@ -39,7 +39,7 @@ then:
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9"
command:
- - "/bin/bash"

2
taskcluster/test-linux-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
PIP_DEFAULT_TIMEOUT: "60"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9"
command:
- "/bin/bash"

2
taskcluster/test-raspbian-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
PIP_DEFAULT_TIMEOUT: "60"
PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1"
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9"
command:
- "/bin/bash"

2
taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
View File

@ -7,7 +7,7 @@ build:
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu"
convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/convert_graphdef_memmapped_format"
convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/convert_graphdef_memmapped_format"
metadata:
name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version"

12
tc-tests-utils.sh
View File

@ -450,7 +450,7 @@ maybe_ssl102_py37()
do_deepspeech_python_build()
{
rename_to_gpu=$1
rename_to_cuda=$1
unset PYTHON_BIN_PATH
unset PYTHONPATH
@ -463,8 +463,8 @@ do_deepspeech_python_build()
mkdir -p wheels
SETUP_FLAGS=""
if [ "${rename_to_gpu}" ]; then
SETUP_FLAGS="--project_name deepspeech-gpu"
if [ "${rename_to_cuda}" ]; then
SETUP_FLAGS="--project_name deepspeech-cuda"
fi
for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do
@ -508,7 +508,7 @@ do_deepspeech_python_build()
do_deepspeech_nodejs_build()
{
rename_to_gpu=$1
rename_to_cuda=$1
npm update && npm install node-gyp node-pre-gyp
@ -523,8 +523,8 @@ do_deepspeech_nodejs_build()
clean node-wrapper
done;
if [ "${rename_to_gpu}" ]; then
make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-gpu
if [ "${rename_to_cuda}" ]; then
make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-cuda
else
make -C native_client/javascript clean npm-pack
fi

2
util/taskcluster.py
View File

@ -66,7 +66,7 @@ if __name__ == '__main__':
parser.add_argument('--target', required=True,
help='Where to put the native client binary files')
parser.add_argument('--arch', required=False, default='cpu',
help='Which architecture to download binaries for. "arm" for ARM 7 (32-bit), "gpu" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)')
help='Which architecture to download binaries for. "arm" for ARMv7 (RPi3/Raspbian), "arm64" for Aarch64 (LePotato/ARMbian), "cuda" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)')
parser.add_argument('--artifact', required=False,
default='native_client.tar.xz',
help='Name of the artifact to download. Defaults to "native_client.tar.xz"')