From bd0fedc312d7baf42dca6f311fba97a2a4191732 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 19 Oct 2017 11:46:37 +0200 Subject: [PATCH] Split TaskCluster build scripts and refactor some variables This will help landing AOT support, avoiding turning tc-build.sh and tc-package.sh into a spaghetti mess. Fixes #898 --- taskcluster/.build.yml | 8 +- taskcluster/.shared.yml | 5 + taskcluster/cuda-build.sh | 24 +++ taskcluster/cuda-package.sh | 9 + taskcluster/darwin-amd64-cpu-opt.yml | 4 + taskcluster/darwin-opt-base.tyml | 20 ++- taskcluster/decoder-build.sh | 13 ++ taskcluster/decoder-package.sh | 10 ++ taskcluster/host-build.sh | 27 +++ taskcluster/linux-amd64-cpu-opt.yml | 4 + taskcluster/linux-amd64-ctc-opt.yml | 10 ++ taskcluster/linux-amd64-gpu-opt.yml | 6 +- taskcluster/linux-opt-base.tyml | 22 ++- taskcluster/linux-rpi3-cpu-opt.yml | 8 +- taskcluster/package.sh | 17 ++ taskcluster/rpi3-build.sh | 23 +++ .../test-benchmark-linux-amd64-opt.yml | 2 +- taskcluster/test-cpp-linux-amd64-opt.yml | 2 +- taskcluster/test-linux-opt-base.tyml | 12 +- .../test-nodejs_4x-linux-amd64-opt.yml | 4 +- .../test-nodejs_5x-linux-amd64-opt.yml | 4 +- .../test-nodejs_6x-linux-amd64-opt.yml | 4 +- .../test-python_27-linux-amd64-opt.yml | 2 +- .../test-python_34-linux-amd64-opt.yml | 2 +- .../test-python_35-linux-amd64-opt.yml | 2 +- .../test-python_36-linux-amd64-opt.yml | 2 +- taskcluster/test-training-linux-amd64-opt.yml | 4 +- tc-build.sh | 101 ----------- tc-package.sh | 39 ----- tc-tests-utils.sh | 165 +++++++++++++++++- tc-train-tests.sh | 2 +- 31 files changed, 365 insertions(+), 192 deletions(-) create mode 100755 taskcluster/cuda-build.sh create mode 100755 taskcluster/cuda-package.sh create mode 100755 taskcluster/decoder-build.sh create mode 100755 taskcluster/decoder-package.sh create mode 100755 taskcluster/host-build.sh create mode 100644 taskcluster/linux-amd64-ctc-opt.yml create mode 100755 taskcluster/package.sh create mode 100755 taskcluster/rpi3-build.sh delete mode 100755 tc-build.sh delete mode 100755 tc-package.sh diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 721a3b23..1367f9b3 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -9,9 +9,11 @@ build: true git_lfs: - > - export PATH=/home/build-user/bin:$PATH && + export PATH=${system.homedir.linux}/bin:$PATH && wget https://github.com/git-lfs/git-lfs/releases/download/v2.3.1/git-lfs-linux-amd64-2.3.1.tar.gz -O - | tar -C /tmp -zxf - && - PREFIX=/home/build-user/ /tmp/git-lfs-2.3.1/install.sh + PREFIX=${system.homedir.linux}/ /tmp/git-lfs-2.3.1/install.sh + scripts: + build: '' + package: '' args: - tcbuild: '' tests_cmdline: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index de0ed5dc..fa9b94d8 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -7,3 +7,8 @@ nodejs: prep_4: 'echo "deb http://deb.nodesource.com/node_4.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_5: 'echo "deb http://deb.nodesource.com/node_5.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_6: 'echo "deb http://deb.nodesource.com/node_6.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' +system: + username: 'build-user' + homedir: + linux: '/home/build-user' + osx: '/Users/build-user' diff --git a/taskcluster/cuda-build.sh b/taskcluster/cuda-build.sh new file mode 100755 index 00000000..ddd8648d --- /dev/null +++ b/taskcluster/cuda-build.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh + +BAZEL_TARGETS=" +//native_client:deepspeech +//native_client:deepspeech_utils +//native_client:generate_trie +${BAZEL_CTC_TARGETS} +" + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_OPT_FLAGS}" +SYSTEM_TARGET=host +EXTRA_LOCAL_CFLAGS="" +EXTRA_LOCAL_LDFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/ -lcudart -lcuda" + +do_bazel_build + +do_deepspeech_binary_build diff --git a/taskcluster/cuda-package.sh b/taskcluster/cuda-package.sh new file mode 100755 index 00000000..1a62f4d7 --- /dev/null +++ b/taskcluster/cuda-package.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +package_native_client "native_client.tar.xz" diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 37fd47e5..c78f0399 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -5,6 +5,10 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.osx" - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.osx/artifacts/public/summarize_graph" + scripts: + build: "taskcluster/host-build.sh" + package: "taskcluster/package.sh" maxRunTime: 14400 metadata: name: "DeepSpeech OSX AMD64 CPU" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index 66bcd277..a009d12e 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -35,7 +35,13 @@ payload: maxRunTime: { $eval: to_int(build.maxRunTime) } env: - TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + $let: + training: { $eval: as_slugid("test-training-linux-amd64-opt") } + in: + TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph} + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/runs/0/artifacts/public/output_graph.pb + DEEPSPEECH_PROD_MODEL: https://github.com/lissyx/DeepSpeech/releases/download/0.0.2/tc-fake-prod.988_e120.LSTM.ldc93s1.pb # There is no VM yet running tasks on OSX # so one should install by hand: @@ -50,9 +56,9 @@ payload: # gtar needed for --transform # --transform used to rewrite symlinks # this rewrite should take care of tranforming - # - /Users/build-user/TaskCluster/Tasks/task_[[:digit:]]\{10\}/ - # - /Users/build-user/TaskCluster/HeavyTasks/X/task_[[:digit:]]\{10\}/ - # - /Users/build-user/TaskCluster/LightTasks/X/task_[[:digit:]]\{10\}/ + # - ${system.homedir.osx}/TaskCluster/Tasks/task_[[:digit:]]\{10\}/ + # - ${system.homedir.osx}/TaskCluster/HeavyTasks/X/task_[[:digit:]]\{10\}/ + # - ${system.homedir.osx}/TaskCluster/LightTasks/X/task_[[:digit:]]\{10\}/ # into # - $TASKCLUSTER_TASK_DIR/ # => goal: @@ -75,7 +81,7 @@ payload: export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && export TASKCLUSTER_TASK_DIR="$(pwd)" && export LC_ALL=C && - export TASKCLUSTER_ORIGIN_PATH="/Users/build-user/TaskCluster/.*/task_[[:digit:]]\{10\}" && + export TASKCLUSTER_ORIGIN_PATH="${system.homedir.osx}/TaskCluster/.*/task_[[:digit:]]\{10\}" && export TASKCLUSTER_REWRITE_PATH="$TASKCLUSTER_TASK_DIR" && export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" && env && @@ -125,8 +131,8 @@ payload: ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && cd $TASKCLUSTER_TASK_DIR && $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/tc-brew.sh && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-build.sh && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-package.sh + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} artifacts: - type: "directory" diff --git a/taskcluster/decoder-build.sh b/taskcluster/decoder-build.sh new file mode 100755 index 00000000..d177a567 --- /dev/null +++ b/taskcluster/decoder-build.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS}" +BAZEL_TARGETS="${BAZEL_CTC_TARGETS}" + +do_bazel_build diff --git a/taskcluster/decoder-package.sh b/taskcluster/decoder-package.sh new file mode 100755 index 00000000..40e63d47 --- /dev/null +++ b/taskcluster/decoder-package.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +tar -cf - \ + -C ${DS_TFDIR}/bazel-bin/native_client/ libctc_decoder_with_kenlm.so | pixz -9 > "${TASKCLUSTER_ARTIFACTS}/decoder.tar.xz" diff --git a/taskcluster/host-build.sh b/taskcluster/host-build.sh new file mode 100755 index 00000000..009cda76 --- /dev/null +++ b/taskcluster/host-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh + +BAZEL_TARGETS=" +//native_client:deepspeech +//native_client:deepspeech_utils +//native_client:generate_trie +${BAZEL_CTC_TARGETS} +" + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET=host + +BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS}" + +do_bazel_build + +do_deepspeech_binary_build + +do_deepspeech_python_build + +do_deepspeech_nodejs_build diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 2e80f56f..a643d64d 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -16,6 +16,10 @@ build: wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig3.0_3.0.10-1.1_amd64.deb -O /tmp/swig3.0_3.0.10-1.1_amd64.deb && dpkg -i /tmp/swig_3.0.10-1.1_amd64.deb /tmp/swig3.0_3.0.10-1.1_amd64.deb tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/summarize_graph" + scripts: + build: "taskcluster/host-build.sh" + package: "taskcluster/package.sh" metadata: name: "DeepSpeech Linux AMD64 CPU" description: "Building DeepSpeech for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml new file mode 100644 index 00000000..3cc6fbb6 --- /dev/null +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -0,0 +1,10 @@ +build: + template_file: linux-opt-base.tyml + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/summarize_graph" + scripts: + build: 'taskcluster/decoder-build.sh' + package: 'taskcluster/decoder-package.sh' + metadata: + name: "DeepSpeech CTC KenLM Decoder Linux AMD64 CPU" + description: "Building DeepSpeech CTC KenLM Decoder for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 293355ff..867a7ca2 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -5,9 +5,11 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.gpu" - "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}" tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.gpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.gpu/artifacts/public/summarize_graph" maxRunTime: 14400 - args: - tcbuild: "--gpu" + scripts: + build: "taskcluster/cuda-build.sh" + package: "taskcluster/cuda-package.sh" metadata: name: "DeepSpeech Linux AMD64 CUDA" description: "Building DeepSpeech for Linux/AMD64, CUDA-enabled, optimized version" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 5fd9e1b2..80c472af 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -30,7 +30,13 @@ payload: image: "ubuntu:14.04" env: - TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + $let: + training: { $eval: as_slugid("test-training-linux-amd64-opt") } + in: + TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + SUMMARIZE_GRAPH_BINARY: ${build.summarize_graph} + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/runs/0/artifacts/public/output_graph.pb + DEEPSPEECH_PROD_MODEL: https://github.com/lissyx/DeepSpeech/releases/download/0.0.2/tc-fake-prod.988_e120.LSTM.ldc93s1.pb command: - "/bin/bash" @@ -42,13 +48,13 @@ payload: installGitlfs: { $eval: strip(str(build.git_lfs)) } in: > apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} && - adduser --system --home /home/build-user build-user && - cd /home/build-user/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C /home/build-user/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u build-user /bin/bash /tmp/clone.sh && - /home/build-user/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} && - sudo -H -u build-user /bin/bash /home/build-user/DeepSpeech/ds/tc-build.sh ${build.args.tcbuild} && - sudo -H -u build-user /bin/bash /home/build-user/DeepSpeech/ds/tc-package.sh + adduser --system --home ${system.homedir.linux} ${system.username} && + cd ${system.homedir.linux}/ && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + ${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && + sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} artifacts: "public": diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 86cff853..d17f4e60 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -5,6 +5,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: - > @@ -13,9 +14,10 @@ build: echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb system_config: - > - multistrap -d /tmp/multistrap-raspbian-jessie/ -f /home/build-user/DeepSpeech/ds/native_client/multistrap.conf - args: - tcbuild: "--arm" + multistrap -d /tmp/multistrap-raspbian-jessie/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap.conf + scripts: + build: "taskcluster/rpi3-build.sh" + package: "taskcluster/package.sh" metadata: name: "DeepSpeech Linux RPi3/ARMv6 CPU" description: "Building DeepSpeech for Linux RPi3 ARMv6, CPU only, optimized version" diff --git a/taskcluster/package.sh b/taskcluster/package.sh new file mode 100755 index 00000000..d7ddd386 --- /dev/null +++ b/taskcluster/package.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +package_native_client "native_client.tar.xz" + +if [ -d ${DS_ROOT_TASK}/DeepSpeech/ds/wheels ]; then + # Python wheels + cp ${DS_ROOT_TASK}/DeepSpeech/ds/wheels/* ${TASKCLUSTER_ARTIFACTS}/ + + # NodeJS package + cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/deepspeech-*.tgz ${TASKCLUSTER_ARTIFACTS}/ +fi; diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh new file mode 100755 index 00000000..ad17d588 --- /dev/null +++ b/taskcluster/rpi3-build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/../tc-tests-utils.sh + +source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh + +BAZEL_TARGETS=" +//native_client:deepspeech +//native_client:deepspeech_utils +//native_client:generate_trie +${BAZEL_CTC_TARGETS} +" + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET=rpi3 + +BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS}" + +do_bazel_build + +do_deepspeech_binary_build diff --git a/taskcluster/test-benchmark-linux-amd64-opt.yml b/taskcluster/test-benchmark-linux-amd64-opt.yml index 527278fc..70b7d647 100644 --- a/taskcluster/test-benchmark-linux-amd64-opt.yml +++ b/taskcluster/test-benchmark-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > apt-get -qq -y install ${python.packages.apt} zip args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-benchmark-tests.sh" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-benchmark-tests.sh" metadata: name: "DeepSpeech Linux AMD64 CPU benchmarking" description: "Benchmarking DeepSpeech for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-cpp-linux-amd64-opt.yml b/taskcluster/test-cpp-linux-amd64-opt.yml index c757a6e9..f1f45eca 100644 --- a/taskcluster/test-cpp-linux-amd64-opt.yml +++ b/taskcluster/test-cpp-linux-amd64-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-cpu-opt" - "test-training-linux-amd64-opt" args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-cpp-ds-tests.sh" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-cpp-ds-tests.sh" metadata: name: "DeepSpeech Linux AMD64 CPU C++ tests" description: "Testing DeepSpeech C++ for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index f749223b..ccb3ea36 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -25,10 +25,12 @@ then: $let: training: { $eval: as_slugid("test-training-linux-amd64-opt") } linux_amd64_build: { $eval: as_slugid("linux-amd64-cpu-opt") } + linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") } in: TENSORFLOW_WHEEL: https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/tensorflow_warpctc-1.3.0rc0-cp27-cp27mu-linux_x86_64.whl DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/runs/0/artifacts/public - DEEPSPEECH_MODEL: https://queue.taskcluster.net/v1/task/${training}/runs/0/artifacts/public/output_graph.pb + DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/runs/0/artifacts/public/decoder.tar.xz + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/runs/0/artifacts/public/output_graph.pb command: - "/bin/bash" @@ -39,11 +41,11 @@ then: installGitlfs: { $eval: strip(str(build.git_lfs)) } in: > apt-get -qq update && apt-get -qq -y install git pixz libsox2 wget && ${extraSystemSetup} && - adduser --system --home /home/build-user build-user && - cd /home/build-user && + adduser --system --home ${system.homedir.linux} ${system.username} && + cd ${system.homedir.linux} && echo -e "#!/bin/bash\nset -xe\n ${installGitlfs} env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u build-user /bin/bash /tmp/clone.sh && - sudo -H -u build-user --preserve-env /bin/bash ${build.args.tests_cmdline} + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline} artifacts: "public": diff --git a/taskcluster/test-nodejs_4x-linux-amd64-opt.yml b/taskcluster/test-nodejs_4x-linux-amd64-opt.yml index cd0d83b3..bf168074 100644 --- a/taskcluster/test-nodejs_4x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_4x-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > ${nodejs.packages.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-node-tests.sh 4.x" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 4.x" metadata: - name: "DeepSpeech Linux AMD64 CPU on NodeJS 4.x tests" + name: "DeepSpeech Linux AMD64 CPU NodeJS 4.x tests" description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v4.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_5x-linux-amd64-opt.yml b/taskcluster/test-nodejs_5x-linux-amd64-opt.yml index 9ce5e72d..df52dc07 100644 --- a/taskcluster/test-nodejs_5x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_5x-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > ${nodejs.packages.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-node-tests.sh 5.x" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 5.x" metadata: - name: "DeepSpeech Linux AMD64 CPU on NodeJS 5.x tests" + name: "DeepSpeech Linux AMD64 CPU NodeJS 5.x tests" description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v5.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_6x-linux-amd64-opt.yml b/taskcluster/test-nodejs_6x-linux-amd64-opt.yml index a36c2cca..4d40d696 100644 --- a/taskcluster/test-nodejs_6x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_6x-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-node-tests.sh 6.x" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 6.x" metadata: - name: "DeepSpeech Linux AMD64 CPU on NodeJS 6.x tests" + name: "DeepSpeech Linux AMD64 CPU NodeJS 6.x tests" description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v6.x, CPU only, optimized version" diff --git a/taskcluster/test-python_27-linux-amd64-opt.yml b/taskcluster/test-python_27-linux-amd64-opt.yml index 53e56f5c..e9cde33b 100644 --- a/taskcluster/test-python_27-linux-amd64-opt.yml +++ b/taskcluster/test-python_27-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > apt-get -qq -y install ${python.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-python-tests.sh 2.7.13" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 2.7.13" metadata: name: "DeepSpeech Linux AMD64 CPU Python v2.7 tests" description: "Testing DeepSpeech for Linux/AMD64 on Python v2.7, CPU only, optimized version" diff --git a/taskcluster/test-python_34-linux-amd64-opt.yml b/taskcluster/test-python_34-linux-amd64-opt.yml index aee9d282..7edd9c82 100644 --- a/taskcluster/test-python_34-linux-amd64-opt.yml +++ b/taskcluster/test-python_34-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > apt-get -qq -y install ${python.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-python-tests.sh 3.4.6" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.4.6" metadata: name: "DeepSpeech Linux AMD64 CPU Python v3.4 tests" description: "Testing DeepSpeech for Linux/AMD64 on Python v3.4, CPU only, optimized version" diff --git a/taskcluster/test-python_35-linux-amd64-opt.yml b/taskcluster/test-python_35-linux-amd64-opt.yml index f77bb595..7c70faf8 100644 --- a/taskcluster/test-python_35-linux-amd64-opt.yml +++ b/taskcluster/test-python_35-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > apt-get -qq -y install ${python.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-python-tests.sh 3.5.3" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.5.3" metadata: name: "DeepSpeech Linux AMD64 CPU Python v3.5 tests" description: "Testing DeepSpeech for Linux/AMD64 on Python v3.5, CPU only, optimized version" diff --git a/taskcluster/test-python_36-linux-amd64-opt.yml b/taskcluster/test-python_36-linux-amd64-opt.yml index f37653ec..1b498f51 100644 --- a/taskcluster/test-python_36-linux-amd64-opt.yml +++ b/taskcluster/test-python_36-linux-amd64-opt.yml @@ -7,7 +7,7 @@ build: - > apt-get -qq -y install ${python.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-python-tests.sh 3.6.2" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.6.2" metadata: name: "DeepSpeech Linux AMD64 CPU Python v3.6 tests" description: "Testing DeepSpeech for Linux/AMD64 on Python v3.6, CPU only, optimized version" diff --git a/taskcluster/test-training-linux-amd64-opt.yml b/taskcluster/test-training-linux-amd64-opt.yml index eaa2d3d2..43c2cfa7 100644 --- a/taskcluster/test-training-linux-amd64-opt.yml +++ b/taskcluster/test-training-linux-amd64-opt.yml @@ -1,12 +1,12 @@ build: template_file: test-linux-opt-base.tyml dependencies: - - "linux-amd64-cpu-opt" + - "linux-amd64-ctc-opt" system_setup: - > apt-get -qq -y install ${python.packages.apt} args: - tests_cmdline: "/home/build-user/DeepSpeech/ds/tc-train-tests.sh 2.7.13" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13" metadata: name: "DeepSpeech Linux AMD64 CPU training" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64, CPU only, optimized version" diff --git a/tc-build.sh b/tc-build.sh deleted file mode 100755 index 41749c6e..00000000 --- a/tc-build.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -set -xe - -source $(dirname "$0")/tc-tests-utils.sh - -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh - -DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/tf -EXTRA_CUDA_CFLAGS= -EXTRA_CUDA_LDFLAGS= - -if [ "$1" = "--gpu" ]; then - BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" - BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_OPT_FLAGS}" - SYSTEM_TARGET=host - EXTRA_CUDA_CFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/" - EXTRA_CUDA_LDFLAGS="-lcudart -lcuda" -fi - -if [ "$1" = "--arm" ]; then - BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" - BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS}" - SYSTEM_TARGET=rpi3 -fi - -if [ "$1" != "--gpu" -a "$1" != "--arm" ]; then - BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" - BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS}" - SYSTEM_TARGET=host - MAKE_BINDINGS_PY=1 - MAKE_BINDINGS_JS=1 -fi - -cd ${DS_ROOT_TASK}/DeepSpeech/tf -eval "export ${BAZEL_ENV_FLAGS}" -PATH=${DS_ROOT_TASK}/bin/:$PATH bazel ${BAZEL_OUTPUT_USER_ROOT} \ - build -c opt ${BAZEL_BUILD_FLAGS} \ - //native_client:deepspeech \ - //native_client:deepspeech_utils \ - //native_client:ctc_decoder_with_kenlm \ - //native_client:generate_trie - -cd ${DS_ROOT_TASK}/DeepSpeech/ds/ -make -C native_client/ \ - TARGET=${SYSTEM_TARGET} \ - TFDIR=${DS_TFDIR} \ - RASPBIAN=/tmp/multistrap-raspbian-jessie \ - EXTRA_CFLAGS="${EXTRA_CUDA_CFLAGS}" \ - EXTRA_LDFLAGS="${EXTRA_CUDA_LDFLAGS}" \ - deepspeech - -if [ "${OS}" = "Darwin" ]; then - export SWIG_LIB="$(find ${DS_ROOT_TASK}/homebrew/Cellar/swig/ -type f -name "swig.swg" | xargs dirname)" -fi; - -if [ ${MAKE_BINDINGS_PY} ]; then - unset PYTHON_BIN_PATH - unset PYTHONPATH - export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv" - export PATH="${PYENV_ROOT}/bin:$PATH" - - install_pyenv "${PYENV_ROOT}" - install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - - mkdir -p wheels - - for pyver in 2.7.13 3.4.6 3.5.3 3.6.2; do - pyenv install ${pyver} - pyenv virtualenv ${pyver} deepspeech - source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate - - make -C native_client/ \ - TFDIR=${DS_TFDIR} \ - bindings-clean bindings - - cp native_client/dist/deepspeech-*.whl wheels - - make -C native_client/ bindings-clean - - deactivate - pyenv uninstall --force deepspeech - done; -fi - -if [ ${MAKE_BINDINGS_JS} ]; then - npm update && npm install node-gyp node-pre-gyp - - export PATH="$(npm root)/.bin/:$PATH" - - # 7.10.0 and 8.0.0 targets fails to build - # > ../deepspeech_wrap.cxx:966:23: error: 'WeakCallbackData' in namespace 'v8' does not name a type - for node in 4.8.0 5.12.0 6.10.0; do - make -C native_client/javascript \ - TFDIR=${DS_TFDIR} \ - NODE_ABI_TARGET=--target=$node \ - clean package - done; - - make -C native_client/javascript clean npm-pack -fi diff --git a/tc-package.sh b/tc-package.sh deleted file mode 100755 index 2784fcb3..00000000 --- a/tc-package.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -xe - -source $(dirname "$0")/tc-tests-utils.sh - -mkdir -p ${TASKCLUSTER_ARTIFACTS} || true - -tar -C ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/tensorflow/ \ - -cf ${TASKCLUSTER_ARTIFACTS}/native_client.tar \ - libtensorflow_cc.so - -tar -C ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/ \ - -uf ${TASKCLUSTER_ARTIFACTS}/native_client.tar \ - libdeepspeech.so \ - libdeepspeech_utils.so \ - libctc_decoder_with_kenlm.so \ - generate_trie - -tar -C ${DS_ROOT_TASK}/DeepSpeech/ds/ \ - -uf ${TASKCLUSTER_ARTIFACTS}/native_client.tar \ - LICENSE - -tar -C ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/ \ - -uf ${TASKCLUSTER_ARTIFACTS}/native_client.tar \ - deepspeech - -tar -C ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/kenlm/ \ - -uf ${TASKCLUSTER_ARTIFACTS}/native_client.tar \ - README.mozilla - -if [ -d ${DS_ROOT_TASK}/DeepSpeech/ds/wheels ]; then - cp ${DS_ROOT_TASK}/DeepSpeech/ds/wheels/* ${TASKCLUSTER_ARTIFACTS}/ -fi - -find ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/ -type f -name "deepspeech-*.tgz" -exec cp {} ${TASKCLUSTER_ARTIFACTS}/ \; - -pixz -9 ${TASKCLUSTER_ARTIFACTS}/native_client.tar ${TASKCLUSTER_ARTIFACTS}/native_client.tar.xz -rm ${TASKCLUSTER_ARTIFACTS}/native_client.tar diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index cb2503d1..d4192cda 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -9,11 +9,16 @@ fi; if [ "${OS}" = "Darwin" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export SWIG_LIB="$(find ${DS_ROOT_TASK}/homebrew/Cellar/swig/ -type f -name "swig.swg" | xargs dirname)" fi; export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} +export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/tf +export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds -model_name=$(basename "${DEEPSPEECH_MODEL}") +export BAZEL_CTC_TARGETS="//native_client:ctc_decoder_with_kenlm" + +model_name=$(basename "${DEEPSPEECH_TEST_MODEL}") assert_correct_inference() { @@ -24,13 +29,13 @@ assert_correct_inference() echo "One or more empty strings:" echo "phrase: <${phrase}>" echo "expected: <${expected}>" - exit 1; + return 1 fi; if [ "${phrase}" = "${expected}" ]; then echo "Proper output has been produced:" echo "${phrase}" - exit 0 + return 0 else echo "!! Non matching output !!" echo "got: <${phrase}>" @@ -38,7 +43,7 @@ assert_correct_inference() echo "-------------------" echo "expected: <${expected}>" echo "xxd:"; echo "${expected}" | xxd - exit 1 + return 1 fi; } @@ -47,23 +52,36 @@ assert_correct_ldc93s1() assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" } -download_native_client_files() +generic_download_tarxz() { target_dir=$1 + url=$2 - if [ -z "${target_dir}" ]; then - echo "Empty name for target directory: ${target_dir}" + if [ -z "${target_dir}" -o -z "${url}" ]; then + echo "Empty name for target directory or URL:" + echo " target_dir=${target_dir}" + echo " url=${url}" exit 1 fi; mkdir -p ${target_dir} || true - wget ${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz -O - | pixz -d | tar -C ${target_dir} -xf - + wget ${url} -O - | pixz -d | tar -C ${target_dir} -xf - +} + +download_native_client_files() +{ + generic_download_tarxz "$1" "${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz" +} + +download_ctc_kenlm() +{ + generic_download_tarxz "$1" "${DEEPSPEECH_LIBCTC}" } download_data() { - wget ${DEEPSPEECH_MODEL} -O /tmp/${model_name} + wget ${DEEPSPEECH_TEST_MODEL} -O /tmp/${model_name} wget https://catalog.ldc.upenn.edu/desc/addenda/LDC93S1.wav -O /tmp/LDC93S1.wav cp ~/DeepSpeech/ds/data/alphabet.txt /tmp/alphabet.txt cp ~/DeepSpeech/ds/data/lm/lm.binary /tmp/lm.binary @@ -107,3 +125,132 @@ install_pyenv_virtualenv() popd eval "$(pyenv virtualenv-init -)" } + +do_get_model_parameters() +{ + local __result=$2 + model_url=$1 + model_file=/tmp/$(basename "${model_url}") + + if [ -z "${model_url}" ]; then + echo "Empty URL for model" + exit 1 + fi; + + wget "${model_url}" -O "${model_file}" + wget "${SUMMARIZE_GRAPH_BINARY}" -O "/tmp/summarize_graph" + + if [ "${OS}" = "Darwin" ]; then + mv /tmp/summarize_graph /tmp/summarize_graph.gz && gunzip /tmp/summarize_graph.gz + fi; + chmod +x /tmp/summarize_graph + + if [ ! -f "${model_file}" ]; then + echo "No such model: ${model_file}" + exit 1 + fi; + + model_width=$(/tmp/summarize_graph --in_graph="${model_file}" | grep "inputs" | grep -Eo "shape=\[\?,\?,[[:digit:]]+" | cut -d',' -f3) + + eval $__result="'--define=DS_MODEL_FRAMESIZE=${model_width} --define=DS_MODEL_FILE=${model_file}'" +} + +do_bazel_build() +{ + cd ${DS_ROOT_TASK}/DeepSpeech/tf + eval "export ${BAZEL_ENV_FLAGS}" + PATH=${DS_ROOT_TASK}/bin/:$PATH bazel ${BAZEL_OUTPUT_USER_ROOT} build \ + -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} +} + +do_deepspeech_binary_build() +{ + cd ${DS_DSDIR} + make -C native_client/ \ + TARGET=${SYSTEM_TARGET} \ + TFDIR=${DS_TFDIR} \ + RASPBIAN=/tmp/multistrap-raspbian-jessie \ + EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ + EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \ + EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \ + deepspeech +} + +do_deepspeech_python_build() +{ + unset PYTHON_BIN_PATH + unset PYTHONPATH + export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv" + export PATH="${PYENV_ROOT}/bin:$PATH" + + install_pyenv "${PYENV_ROOT}" + install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" + + mkdir -p wheels + + for pyver in 2.7.13 3.4.6 3.5.3 3.6.2; do + pyenv install ${pyver} + pyenv virtualenv ${pyver} deepspeech + source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate + + EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" make -C native_client/ \ + TFDIR=${DS_TFDIR} \ + bindings-clean bindings + + cp native_client/dist/deepspeech-*.whl wheels + + make -C native_client/ bindings-clean + + deactivate + pyenv uninstall --force deepspeech + done; +} + +do_deepspeech_nodejs_build() +{ + npm update && npm install node-gyp node-pre-gyp + + export PATH="$(npm root)/.bin/:$PATH" + + # 7.10.0 and 8.0.0 targets fails to build + # > ../deepspeech_wrap.cxx:966:23: error: 'WeakCallbackData' in namespace 'v8' does not name a type + for node in 4.8.0 5.12.0 6.10.0; do + EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" make -C native_client/javascript \ + TFDIR=${DS_TFDIR} \ + NODE_ABI_TARGET=--target=$node \ + clean package + done; + + make -C native_client/javascript clean npm-pack +} + +package_native_client() +{ + tensorflow_dir=${DS_TFDIR} + deepspeech_dir=${DS_DSDIR} + artifacts_dir=${TASKCLUSTER_ARTIFACTS} + artifact_name=$1 + + if [ ! -d ${tensorflow_dir} -o ! -d ${deepspeech_dir} -o ! -d ${artifacts_dir} ]; then + echo "Missing directory. Please check:" + echo "tensorflow_dir=${tensorflow_dir}" + echo "deepspeech_dir=${deepspeech_dir}" + echo "artifacts_dir=${artifacts_dir}" + exit 1 + fi; + + if [ -z "${artifact_name}" ]; then + echo "Please specify artifact name." + fi; + + tar -cf - \ + -C ${tensorflow_dir}/bazel-bin/tensorflow/ libtensorflow_cc.so \ + -C ${tensorflow_dir}/bazel-bin/native_client/ generate_trie \ + -C ${tensorflow_dir}/bazel-bin/native_client/ libctc_decoder_with_kenlm.so \ + -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \ + -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech_utils.so \ + -C ${deepspeech_dir}/ LICENSE \ + -C ${deepspeech_dir}/native_client/ deepspeech \ + -C ${deepspeech_dir}/native_client/kenlm/ README.mozilla \ + | pixz -9 > "${artifacts_dir}/${artifact_name}" +} diff --git a/tc-train-tests.sh b/tc-train-tests.sh index c0c4e843..757c6b3e 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -31,7 +31,7 @@ source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate pip install --upgrade ${TENSORFLOW_WHEEL} grep -v "tensorflow" ${HOME}/DeepSpeech/ds/requirements.txt | pip install --upgrade -r /dev/stdin -download_native_client_files "/tmp/ds" +download_ctc_kenlm "/tmp/ds" pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1.sh