diff --git a/.install b/.install index bb47ce32..3180f6e7 100755 --- a/.install +++ b/.install @@ -3,8 +3,8 @@ virtualenv -p python3 ../tmp/venv source ../tmp/venv/bin/activate pip install -r <(grep -v tensorflow requirements.txt) -pip install tensorflow-gpu==1.11.0 +pip install tf-nightly==1.12.0.dev20181004 -python3 util/taskcluster.py --arch gpu --target ../tmp/native_client +python3 util/taskcluster.py --arch cuda --target ../tmp/native_client mkdir -p ../keep/summaries diff --git a/DeepSpeech.py b/DeepSpeech.py index 01e80996..dd4ef113 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -1646,6 +1646,9 @@ def train(server=None): update_progressbar.current_job_index += 1 + # Initialize update_progressbar()'s child fields to safe values + update_progressbar.pbar = None + # The MonitoredTrainingSession takes care of session initialization, # restoring from a checkpoint, saving to a checkpoint, and closing when done # or an error occurs. diff --git a/Dockerfile b/Dockerfile index 72aee7a7..d1503d17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -190,7 +190,7 @@ RUN cp /tensorflow/bazel-bin/native_client/libctc_decoder_with_kenlm.so /DeepSpe # Install TensorFlow WORKDIR /DeepSpeech/ -RUN pip install tensorflow-gpu==1.11.0 +RUN pip install tf-nightly==1.12.0.dev20181004 # Make DeepSpeech and install Python bindings diff --git a/bin/benchmark_nc.py b/bin/benchmark_nc.py index 4160ca83..3171b633 100755 --- a/bin/benchmark_nc.py +++ b/bin/benchmark_nc.py @@ -84,7 +84,7 @@ def get_arch_string(): nv_rc, nv_stdout, nv_stderr = exec_command('nvidia-smi') nv_stdout = nv_stdout.lower().strip() if 'NVIDIA-SMI' in nv_stdout: - return 'gpu' + return 'cuda' else: return 'cpu' diff --git a/native_client/README.md b/native_client/README.md index 939529e9..8c1f9681 100644 --- a/native_client/README.md +++ b/native_client/README.md @@ -17,7 +17,7 @@ python3 util/taskcluster.py --branch "v0.2.0-alpha.6" This will download and extract `native_client.tar.xz` which includes the deepspeech binary and associated libraries as well as the custom decoder OP. `taskcluster.py` will download binaries for the architecture of the host by default, but you can override that behavior with the `--arch` parameter. See the help info with `python util/taskcluster.py -h` for more details. -If you want the CUDA capable version of the binaries, use `--arch gpu`. Note that for now we don't publish CUDA-capable macOS binaries. +If you want the CUDA capable version of the binaries, use `--arch cuda`. Note that TensorFlow does not support CUDA on macOS anymore. If you're looking to train a model, you now have a `libctc_decoder_with_kenlm.so` file that you can pass to the `--decoder_library_path` parameter of `DeepSpeech.py`. diff --git a/requirements.txt b/requirements.txt index 5df2bfb0..90a1ec49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pandas progressbar2 python-utils -tensorflow == 1.11.0 +tf-nightly==1.12.0.dev20181004 numpy matplotlib scipy diff --git a/taskcluster/cuda-build.sh b/taskcluster/cuda-build.sh index 3ccf5769..b10ab307 100755 --- a/taskcluster/cuda-build.sh +++ b/taskcluster/cuda-build.sh @@ -21,8 +21,8 @@ do_bazel_build do_deepspeech_binary_build -do_deepspeech_python_build rename_to_gpu +do_deepspeech_python_build rename_to_cuda -do_deepspeech_nodejs_build rename_to_gpu +do_deepspeech_nodejs_build rename_to_cuda $(dirname "$0")/decoder-build.sh diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index d5fb0fb8..e549f45c 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -6,8 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.osx/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 98c6fccf..d5cb71cf 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -14,8 +14,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index 86b03833..93194f70 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,8 +4,8 @@ build: - "pull_request.synchronize" - "pull_request.reopened" template_file: linux-opt-base.tyml - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-cuda-opt.yml similarity index 70% rename from taskcluster/linux-amd64-gpu-opt.yml rename to taskcluster/linux-amd64-cuda-opt.yml index 25aab6de..38c2a087 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-cuda-opt.yml @@ -1,9 +1,9 @@ build: template_file: linux-opt-base.tyml routes: - - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu" - - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu" - - "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cuda" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cuda" + - "index.project.deepspeech.deepspeech.native_client.cuda.${event.head.sha}" system_setup: > ${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} @@ -12,8 +12,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.gpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.gpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cuda/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cuda/artifacts/public/summarize_graph" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 0a59f2fe..98ea1d71 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.arm64/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.arm64/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index bbda1c5e..ac03008b 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index 8fe1f85f..9fe6d9ef 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -16,8 +16,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/scriptworker-task.yml b/taskcluster/scriptworker-task.yml index a7949636..66d98ab6 100644 --- a/taskcluster/scriptworker-task.yml +++ b/taskcluster/scriptworker-task.yml @@ -5,7 +5,7 @@ build: # - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org - "darwin-amd64-cpu-opt" - "linux-amd64-cpu-opt" - - "linux-amd64-gpu-opt" + - "linux-amd64-cuda-opt" - "linux-rpi3-cpu-opt" - "node-package" allowed: @@ -18,18 +18,18 @@ build: python: - "darwin-amd64-cpu-opt" - "linux-amd64-cpu-opt" - - "linux-amd64-gpu-opt" + - "linux-amd64-cuda-opt" - "linux-rpi3-cpu-opt" # - "linux-arm64-cpu-opt" Aarch64 packages are refused by upload.pypi.org javascript: # GPU package - - "linux-amd64-gpu-opt" + - "linux-amd64-cuda-opt" # CPU package with all archs - "node-package" cpp: - "darwin-amd64-cpu-opt" - "linux-amd64-cpu-opt" - - "linux-amd64-gpu-opt" + - "linux-amd64-cuda-opt" - "linux-rpi3-cpu-opt" - "linux-arm64-cpu-opt" metadata: diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index 33fc3711..8456195f 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9" command: - "/bin/bash" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 8c634320..a8499df3 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -39,7 +39,7 @@ then: DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9" command: - - "/bin/bash" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 8d75d57e..7aaf680c 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -44,7 +44,7 @@ then: DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9" command: - "/bin/bash" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 2ffd35f1..b122b529 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-4603-gee903d1" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5460-g7cd8e9" command: - "/bin/bash" diff --git a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml index 9fe68a22..11f602b2 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu" - convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.ee903d1b348019d28048a5581871fcd8cbb3deff.cpu/artifacts/public/convert_graphdef_memmapped_format" + convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.7cd8e9590457006d9a1be029e69d721b671f6a0c.cpu/artifacts/public/convert_graphdef_memmapped_format" metadata: name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 04b676d4..d90429ba 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -450,7 +450,7 @@ maybe_ssl102_py37() do_deepspeech_python_build() { - rename_to_gpu=$1 + rename_to_cuda=$1 unset PYTHON_BIN_PATH unset PYTHONPATH @@ -463,8 +463,8 @@ do_deepspeech_python_build() mkdir -p wheels SETUP_FLAGS="" - if [ "${rename_to_gpu}" ]; then - SETUP_FLAGS="--project_name deepspeech-gpu" + if [ "${rename_to_cuda}" ]; then + SETUP_FLAGS="--project_name deepspeech-cuda" fi for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do @@ -508,7 +508,7 @@ do_deepspeech_python_build() do_deepspeech_nodejs_build() { - rename_to_gpu=$1 + rename_to_cuda=$1 npm update && npm install node-gyp node-pre-gyp @@ -523,8 +523,8 @@ do_deepspeech_nodejs_build() clean node-wrapper done; - if [ "${rename_to_gpu}" ]; then - make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-gpu + if [ "${rename_to_cuda}" ]; then + make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-cuda else make -C native_client/javascript clean npm-pack fi diff --git a/util/taskcluster.py b/util/taskcluster.py index 360b0117..2e71bcd5 100644 --- a/util/taskcluster.py +++ b/util/taskcluster.py @@ -66,7 +66,7 @@ if __name__ == '__main__': parser.add_argument('--target', required=True, help='Where to put the native client binary files') parser.add_argument('--arch', required=False, default='cpu', - help='Which architecture to download binaries for. "arm" for ARM 7 (32-bit), "gpu" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)') + help='Which architecture to download binaries for. "arm" for ARMv7 (RPi3/Raspbian), "arm64" for Aarch64 (LePotato/ARMbian), "cuda" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)') parser.add_argument('--artifact', required=False, default='native_client.tar.xz', help='Name of the artifact to download. Defaults to "native_client.tar.xz"')