diff --git a/.taskcluster.yml b/.taskcluster.yml index eddc5533..48feee3c 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -27,6 +27,7 @@ tasks: "queue:create-task:lowest:{{ taskcluster.docker.provisionerId }}/deepspeech-worker", "queue:create-task:lowest:deepspeech-provisioner/ds-macos-light", "queue:create-task:lowest:deepspeech-provisioner/ds-scriptworker", + "queue:create-task:lowest:deepspeech-provisioner/ds-rpi3", "queue:route:index.project.deepspeech.*", "queue:route:notify.irc-channel.*", "queue:scheduler-id:taskcluster-github", diff --git a/DeepSpeech.py b/DeepSpeech.py index cf944aee..4b285446 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -1579,7 +1579,7 @@ def train(server=None): is_chief=is_chief, hooks=hooks, checkpoint_dir=FLAGS.checkpoint_dir, - save_checkpoint_secs=FLAGS.checkpoint_secs if FLAGS.train else None, + save_checkpoint_secs=None, # already taken care of by a hook config=session_config) as session: if len(FLAGS.initialize_from_frozen_model) > 0: log_info('Initializing from frozen model: {}'.format(FLAGS.initialize_from_frozen_model)) diff --git a/README.md b/README.md index ddbbaaf5..4e7a3e0a 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,21 @@ Project DeepSpeech is an open source Speech-To-Text engine, using a model traine ![Usage](images/usage.gif) -Pre-built binaries that can be used for performing inference with a trained model can be installed with `pip`. Proper setup using virtual environment is recommended and you can find that documented [below](#using-the-python-package). +Pre-built binaries that can be used for performing inference with a trained model can be installed with `pip3`. Proper setup using virtual environment is recommended and you can find that documented [below](#using-the-python-package). A pre-trained English model is available for use, and can be downloaded using [the instructions below](#getting-the-pre-trained-model). Once everything is installed you can then use the `deepspeech` binary to do speech-to-text on short, approximately 5 second, audio files (currently only WAVE files with 16-bit, 16 kHz, mono are supported in the Python client): ```bash -pip install deepspeech +pip3 install deepspeech deepspeech models/output_graph.pb models/alphabet.txt my_audio_file.wav ``` Alternatively, quicker inference (The realtime factor on a GeForce GTX 1070 is about 0.44.) can be performed using a supported NVIDIA GPU on Linux. (See the release notes to find which GPU's are supported.) This is done by instead installing the GPU specific package: ```bash -pip install deepspeech-gpu +pip3 install deepspeech-gpu deepspeech models/output_graph.pb models/alphabet.txt my_audio_file.wav ``` @@ -52,7 +52,7 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech ## Prerequisites -* [Python 2.7](https://www.python.org/) +* [Python 3.6](https://www.python.org/) * [Git Large File Storage](https://git-lfs.github.com/) ## Getting the code @@ -82,17 +82,17 @@ There are three ways to use DeepSpeech inference: ### Using the Python package -Pre-built binaries that can be used for performing inference with a trained model can be installed with `pip`. You can then use the `deepspeech` binary to do speech-to-text on an audio file: +Pre-built binaries that can be used for performing inference with a trained model can be installed with `pip3`. You can then use the `deepspeech` binary to do speech-to-text on an audio file: -For the Python bindings, it is highly recommended that you perform the installation within a Python 2.7 virtual environment. You can find more information about those in [this documentation](http://docs.python-guide.org/en/latest/dev/virtualenvs/). +For the Python bindings, it is highly recommended that you perform the installation within a Python 3.5 or later virtual environment. You can find more information about those in [this documentation](http://docs.python-guide.org/en/latest/dev/virtualenvs/). We will continue under the assumption that you already have your system properly setup to create new virtual environments. #### Create a DeepSpeech virtual environment -In creating a virtual environment you will create a directory containing a `python` binary and everything needed to run deepspeech. You can use whatever directory you want. For the purpose of the documentation, we will rely on `$HOME/tmp/deepspeech-venv`. You can create it using this command: +In creating a virtual environment you will create a directory containing a `python3` binary and everything needed to run deepspeech. You can use whatever directory you want. For the purpose of the documentation, we will rely on `$HOME/tmp/deepspeech-venv`. You can create it using this command: ``` -$ virtualenv $HOME/tmp/deepspeech-venv/ +$ virtualenv -p python3 $HOME/tmp/deepspeech-venv/ ``` Once this command completes successfully, the environment will be ready to be activated. @@ -107,26 +107,26 @@ $ source $HOME/tmp/deepspeech-venv/bin/activate #### Installing DeepSpeech Python bindings -Once your environment has been setup and loaded, you can use `pip` to manage packages locally. On a fresh setup of the virtualenv, you will have to install the DeepSpeech wheel. You can check if it is already installed by taking a look at the output of `pip list`. To perform the installation, just issue: +Once your environment has been setup and loaded, you can use `pip3` to manage packages locally. On a fresh setup of the virtualenv, you will have to install the DeepSpeech wheel. You can check if it is already installed by taking a look at the output of `pip3 list`. To perform the installation, just issue: ``` -$ pip install deepspeech +$ pip3 install deepspeech ``` If it is already installed, you can also update it: ``` -$ pip install --upgrade deepspeech +$ pip3 install --upgrade deepspeech ``` Alternatively, if you have a supported NVIDIA GPU on Linux (See the release notes to find which GPU's are supported.), you can install the GPU specific package as follows: ``` -$ pip install deepspeech-gpu +$ pip3 install deepspeech-gpu ``` or update it as follows: ``` -$ pip install --upgrade deepspeech-gpu +$ pip3 install --upgrade deepspeech-gpu ``` In both cases, it should take care of installing all the required dependencies. Once it is done, you should be able to call the sample binary using `deepspeech` on your command-line. @@ -146,13 +146,13 @@ See [client.py](native_client/python/client.py) for an example of how to use the To download the pre-built binaries, use `util/taskcluster.py`: ```bash -python util/taskcluster.py --target . +python3 util/taskcluster.py --target . ``` or if you're on macOS: ```bash -python util/taskcluster.py --arch osx --target . +python3 util/taskcluster.py --arch osx --target . ``` This will download `native_client.tar.xz` which includes the deepspeech binary and associated libraries, and extract it into the current folder. `taskcluster.py` will download binaries for Linux/x86_64 by default, but you can override that behavior with the `--arch` parameter. See the help info with `python util/taskcluster.py -h` for more details. @@ -203,13 +203,13 @@ Install the required dependencies using pip: ```bash cd DeepSpeech -pip install -r requirements.txt +pip3 install -r requirements.txt ``` You'll also need to download `native_client.tar.xz` or build the native client files yourself to get the custom TensorFlow OP needed for decoding the outputs of the neural network. You can use `util/taskcluster.py` to download the files for your architecture: ```bash -python util/taskcluster.py --target . +python3 util/taskcluster.py --target . ``` This will download the native client files for the x86_64 architecture without CUDA support, and extract them into the current folder. If you prefer building the binaries from source, see the [native_client README file](native_client/README.md). We also have binaries with CUDA enabled ("--arch gpu") and for ARM7 ("--arch arm"). @@ -219,8 +219,8 @@ This will download the native client files for the x86_64 architecture without C If you have a capable (Nvidia, at least 8GB of VRAM) GPU, it is highly recommended to install TensorFlow with GPU support. Training will likely be significantly quicker than using the CPU. To enable GPU support, you can do: ```bash -pip uninstall tensorflow -pip install 'tensorflow-gpu==1.6.0' +pip3 uninstall tensorflow +pip3 install 'tensorflow-gpu==1.6.0' ``` ### Common Voice training data @@ -349,7 +349,7 @@ For example, if you want to fine tune the entire graph using your own data in `m ```bash mkdir fine_tuning_checkpoints -python DeepSpeech.py --n_hidden 2048 --initialize_from_frozen_model path/to/model/output_graph.pb --checkpoint_dir fine_tuning_checkpoints --epoch 3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001 +python3 DeepSpeech.py --n_hidden 2048 --initialize_from_frozen_model path/to/model/output_graph.pb --checkpoint_dir fine_tuning_checkpoints --epoch 3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001 ``` Note: the released models were trained with `--n_hidden 2048`, so you need to use that same value when initializing from the release models. diff --git a/native_client/Makefile b/native_client/Makefile index 9593df49..9e1fa80d 100644 --- a/native_client/Makefile +++ b/native_client/Makefile @@ -9,7 +9,7 @@ ### $ make -C native_client/ TARGET=rpi3 TFDIR=../../tensorflow/tensorflow/ ### -.PHONY: clean run bindings +.PHONY: clean run bindings print-toolchain include definitions.mk @@ -32,7 +32,7 @@ bindings-clean: bindings-build: pip install --quiet $(PYTHON_PACKAGES) wheel - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" UTILS_LDFLAGS="-L${TFDIR}/bazel-bin/native_client" MODEL_LIBS="$(LIBS)" $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" UTILS_LDFLAGS="-L${TFDIR}/bazel-bin/native_client" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ @@ -41,7 +41,7 @@ MANIFEST.in: bindings-build bindings-package: MANIFEST.in cat MANIFEST.in rm temp_build/python/*_wrap.o - $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" UTILS_LDFLAGS="-L${TFDIR}/bazel-bin/native_client" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) bindings: bindings-build bindings-package @@ -64,3 +64,6 @@ uninstall: rm -f ${PREFIX}/lib/libdeepspeech_utils.so rm -f ${PREFIX}/lib/libdeepspeech.so rmdir --ignore-fail-on-non-empty ${PREFIX}/lib + +print-toolchain: + @echo $(TOOLCHAIN) diff --git a/native_client/definitions.mk b/native_client/definitions.mk index f7fd90ea..20afe4d9 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -8,6 +8,7 @@ SO_SEARCH ?= $(TFDIR)/bazel-bin/ ifeq ($(TARGET),host) TOOLCHAIN := CFLAGS := +CXXFLAGS := LDFLAGS := SOX_CFLAGS := `pkg-config --cflags sox` SOX_LDFLAGS := `pkg-config --libs sox` @@ -18,9 +19,10 @@ endif endif ifeq ($(TARGET),rpi3) -TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf- +TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc49/bin/arm-linux-gnueabihf- RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-stretch) -CFLAGS := -D_GLIBCXX_USE_CXX11_ABI=0 -isystem $(RASPBIAN)/usr/include -isystem $(RASPBIAN)/usr/include/arm-linux-gnueabihf +CFLAGS := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 -isystem $(RASPBIAN)/usr/include -isystem $(RASPBIAN)/usr/include/arm-linux-gnueabihf +CXXFLAGS := $(CXXFLAGS) LDFLAGS := $(RASPBIAN)/lib/arm-linux-gnueabihf/libc.so.6 -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/ SOX_CFLAGS := @@ -28,7 +30,8 @@ SOX_LDFLAGS := $(RASPBIAN)/usr/lib/arm-linux-gnueabihf/libsox.so PYVER := $(shell python -c "import platform; maj, min, _ = platform.python_version_tuple(); print(maj+'.'+min);") PYTHON_PACKAGES := -NUMPY_INCLUDE := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python$(PYVER)/ +PYTHON_PATH := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python$(PYVER)/plat-arm-linux-gnueabihf/:$(RASPBIAN)/usr/lib/python3/dist-packages/ +NUMPY_INCLUDE := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.5/ PYTHON_PLATFORM_NAME := --plat-name linux_armv7l NODE_PLATFORM_TARGET := --target_arch=arm --target_platform=linux TOOLCHAIN_LDD_OPTS := --root $(RASPBIAN)/ @@ -47,10 +50,11 @@ LDFLAGS_NEEDED := LDFLAGS_RPATH := -Wl,-rpath,@executable_path endif -CFLAGS += $(EXTRA_CFLAGS) -LIBS := -ldeepspeech -ldeepspeech_utils $(EXTRA_LIBS) +CFLAGS += $(EXTRA_CFLAGS) +CXXFLAGS += $(EXTRA_CXXFLAGS) +LIBS := -ldeepspeech -ldeepspeech_utils $(EXTRA_LIBS) LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) -LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS) +LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS) AS := $(TOOLCHAIN)as CC := $(TOOLCHAIN)gcc diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index 43183cdc..0da950ff 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -25,7 +25,7 @@ configure: deepspeech_wrap.cxx package.json $(NODE_BUILD_TOOL) configure $(NODE_BUILD_VERBOSE) build: configure deepspeech_wrap.cxx - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS="$(LIBS)" $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS="$(LIBS)" $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE) copy-deps: build $(call copy_missing_libs,lib/binding/*/*/*/deepspeech.node,lib/binding/*/*/) diff --git a/native_client/multistrap.conf b/native_client/multistrap.conf index 5c9145ac..d8f8ffb5 100644 --- a/native_client/multistrap.conf +++ b/native_client/multistrap.conf @@ -7,8 +7,8 @@ aptsources=Raspbian cleanup=true [Raspbian] -packages=libc6 libc6-dev libstdc++-6-dev linux-libc-dev libpython2.7-dev libpython3.4-dev libsox-dev python-numpy -source=http://mirrordirector.raspbian.org/raspbian/ +packages=libc6 libc6-dev libstdc++-6-dev linux-libc-dev libpython3.4-dev libpython3.5-dev libsox-dev python3-numpy python3-setuptools +source=http://raspbian.raspberrypi.org/raspbian/ keyring=raspbian-archive-keyring components=main suite=stretch diff --git a/native_client/xldd b/native_client/xldd new file mode 100755 index 00000000..ef2ca20f --- /dev/null +++ b/native_client/xldd @@ -0,0 +1,362 @@ +#!/bin/bash + +# ldd drop-in replacement for cross-compilation toolchains. + +# This file is a slightly modified version of xldd.in from +# crosstool-ng 1.22.0 + +# In order to use it, copy it in same directory than other +# toolchain binaries and rename it with same tuple. +# (i.e. /opt/arm-sysmic-linux-gnueabihf/bin/arm-sysmic-linux-gnueabihf-ldd) +# Thus, this will automaticaly detect necessary information +# about your toolchain. + +export LC_ALL=C +version="forked from crosstool-ng 1.22.0" +# Change it to 64 if necessary +bits="32" +sed="${SED:-sed}" +grep="${GREP:-grep}" + +my_name="$( basename "${0}" )" +prefix="${0%-ldd}" +gcc="${prefix}-gcc" +readelf="${prefix}-readelf" +fake_load_addr_root="$((0xdeadbeef))" +fake_load_addr_rpath="$((0xdeadc0de))" +fake_load_addr_sysroot="$((0x8badf00d))" +ld_library_path="/lib:/usr/lib" + +do_error() { + printf "%s: %s\n" "${my_name}" "$*" >&2 +} + +do_opt_error() { + do_error "$@" + printf "Try \`%s --help' for more information\n" "${my_name}" >&2 +} + +do_trace() { + local depth=0 + + [ -z "${CT_XLDD_VERBOSE}" ] && return 0 + + for((depth=0; "${#FUNCNAME[$((depth+1))]}" != 0; depth++)); do :; done + printf "%*s" $((4*(depth-1))) "" >&2 + printf -- "$@" >&2 +} + +show_version() { + # Fake a real ldd, just in case some dumb script would check + cat <<_EOF_ +ldd (crosstool-NG) ${version} +Copyright (C) 2010 "Yann E. MORIN" +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +Licensed under the GPLv2, see the file LICENSES in the top-directory of the +sources for this package. +_EOF_ +} + +show_help() { + cat <<_EOF_ +Usage: ${my_name} [OPTION]... --root DIR FILE... + --help print this help and exit + --version print version information and exit + --root dir treat dir as being the root of the target + -s, --show-system mark libs from the sysroot with a trailing '[*]' + and libs found via RPATH with a trailing '[+]' + +_EOF_ + cat <<_EOF_ |fmt +${my_name} tries to mimick the behavior of a real native ldd, but can be +used in a cross-development environment. Here is how it differs from a +real native ldd: + +If the CT_XLDD_VERBOSE variable is set and non-empty, then ${my_name} will +print a lot of debug messages, explaining how it builds the library +search path, and how each library was found and why. + +The LD_LIBRARY_PATH variable is not used, as it can not reliably be +guessed except at runtime, and we can't run. + +${my_name} does not scan /etc/ld.so.cache, but instead uses /etc/ld.so.conf +(it understands the include directives therein for libces that have that). + +${my_name} also interprets (tries to!) the RPATH/RUNPATH records found in +the dynamic ELF section. Such paths are searched for only relative to +the specified root, not from the sysroot (see below). Also, those paths +are searched for not only for the file they appear in, but also for its +dependencies. + +${my_name} will search the directory specified with --root for libraries +to resolve the NEEDED tags. If --root is not set, then ${my_name} will +use the value in the environment variable \${CT_XLDD_ROOT}. If neither +is set, then this is an error. + +If NEEDED libraries can't be found in the specified root directory, then +${my_name} will also look in the sysroot of the toolchain to see if it +can find them. + +For NEEDED libraries that were found, the output will look like: + libneeded.so => /path/to/libneeded.so (0xloadaddr) + +and for those that were not found, the output will look like: + libneeded.so not found + +The paths are relative to the specified root directory, or to the sysroot +(eg. /lib/libneeded.so, /usr/lib/libneeded.so, and so on...). + +The expected load address 'loadaddr' is a faked address to match the output +of the real ldd, but has no actual meaning (set to some constants for now, +0x8badf00d for libraries from the sysroot, 0xdeadc0de for those found via +the RPATH/RUNPATH records, and 0xdeadbeef for others). +_EOF_ + +# Unimplemeted yet: +# -d, --data-relocs process data relocations +# -r, --function-relocs process data and function relocations +# -u, --unused print unused direct dependencies +# -v, --verbose print all information + +# See also this thread: +# http://sourceware.org/ml/crossgcc/2008-09/msg00057.html +} + +# Parse command line options +root="${CT_XLDD_ROOT}" +show_system= +while true; do + case "${1}" in + --help) + show_help + exit 0 + ;; + --version) + show_version + exit 0 + ;; + --root) + root="$2" + shift + ;; + --root=*) + root="${1#--root=}" + ;; + --show-system|-s) + show_system=1 + ;; + -*) + do_opt_error "unrecognized option \`${1}'" + exit 1 + ;; + *) + break + ;; + esac + shift +done + +# Sanity checks +sysroot="$( "${gcc}" -print-sysroot 2>/dev/null )" +if [ -z "${sysroot}" ]; then + sysroot="$( "${gcc}" -print-file-name=libc.so 2>/dev/null \ + |${sed} -r -e 's:/usr/lib/libc.so$::;' \ + )" +fi +if [ -z "${sysroot}" ]; then + do_error "unable to find sysroot for \`${gcc}'" +fi + +if [ -z "${root}" ]; then + root=${sysroot} +fi +if [ ! -d "${root}" ]; then + do_error "\`${root}': no such file or directory" + exit 1 +fi + +do_report_needed_found() { + local needed="${1}" + local path="${2}" + local origin="${3}" + local loadaddr + local sys + + case "${origin}" in + root) + loadaddr="${fake_load_addr_root}" + ;; + rpath) + loadaddr="${fake_load_addr_rpath}" + if [ -n "${show_system}" ]; then + sys=" [+]" + fi + ;; + sysroot) + loadaddr="${fake_load_addr_sysroot}" + if [ -n "${show_system}" ]; then + sys=" [*]" + fi + ;; + esac + + printf "%8s%s => %s (0x%0*x)%s\n" \ + "" \ + "${needed}" \ + "${path}" \ + "$((bits/4))" \ + "${loadaddr}" \ + "${sys}" +} + +# Search a needed file, scanning ${lib_dir} in the root directory +do_find_needed() { + local needed="${1}" + local -a list + local -a dirs + local found + local where + local base + local d i + + do_trace "Searching for '%s'\n" "${needed}" + + # rpath shall come first! + list=( \ + "rpath:${root}" \ + "root:${root}" \ + "sysroot:${sysroot}" \ + ) + + for i in "${list[@]}"; do + where="${i%%:*}" + base="${i#*:}" + if [ "${where}" = "rpath" ]; then + dirs=( "${search_rpath[@]}" ) + else + dirs=( "${needed_search_path[@]}" ) + fi + for d in "${dirs[@]}"; do + do_trace "-> looking in '%s' (%s)\n" "${d}" "${where}" + if [ -f "${base}${d}/${needed}" ]; then + found="${d}/${needed}" + do_trace "---> found\n" + break 2 + fi + done + done + + if [ -n "${found}" ]; then + do_report_needed_found "${needed}" "${found}" "${where}" + do_process_file "${base}${found}" + else + printf "%8s%s not found\n" "" "${needed}" + fi + + do_trace "Done searching for '%s'\n" "${needed}" +} + +# Scan a file for all NEEDED tags +do_process_file() { + local file="${1}" + local -a save_search_rpath + local n m + local found + + do_trace "Parsing file '%s'\n" "${file}" + + save_search_rpath=( "${search_rpath[@]}" ) + for n in $( "${readelf}" -d "${file}" \ + |"${grep}" -E '\((RPATH|RUNPATH)\)' \ + |"${sed}" -r -e 's/^.*Library r(|un)path:[[:space:]]+\[(.*)\]$/\2/;'\ + ); do + do_trace "-> adding rpath '%s'\n" "${n}" + search_rpath+=( "${n}" ) + done + do_trace ": search path:\n" + for n in "${search_rpath[@]}" "${needed_search_path[@]}"; do + do_trace ": - '%s'\n" "${n}" + done + do_trace ": end search path\n" + + for n in $( "${readelf}" -d "${file}" \ + |"${grep}" -E '\(NEEDED\)' \ + |"${sed}" -r -e 's/^.*Shared library:[[:space:]]+\[([^]]+)\].*/\1/;' \ + ); do + found=0 + for m in "${needed_list[@]}"; do + [ "${n}" = "${m}" ] && found=1 && break + done + if [ ${found} -ne 0 ]; then + do_trace "-> skipping already known dependency '%s'\n" "${n}" + continue + fi + do_trace "-> handling new dependency '%s'\n" "${n}" + needed_list+=( "${n}" ) + do_find_needed "${n}" + do_trace "-> done handling dependency '%s'\n" "${n}" + done + + search_rpath=( "${save_search_rpath[@]}" ) + + do_trace "Finished parsing file '%s'\n" "${file}" +} + +# Recursively scan a /etc/ld.so.conf file +do_scan_etc_ldsoconf() { + local ldsoconf="${1}" + local g + local f + + [ -f "${ldsoconf}" ] || return 0 + do_trace "Parsing ld.so.conf: '%s'\n" "${ldsoconf}" + + while read line; do + case "${line}" in + include\ *) + g="${root}${line#include }" + do_trace "-> handling include directive '%s'\n" "${g}" + for f in ${g}; do + do_scan_etc_ldsoconf "${f}" + done + do_trace "-> finished handling include directive '%s'\n" "${g}" + ;; + \#*|"") + ;; + *) + do_trace "-> adding search dir '%s'\n" "${line}" + needed_search_path+=( "${line}" ) + ;; + esac + done <"${ldsoconf}" + + do_trace "Finished parsing ld.so.conf: '%s'\n" "${ldsoconf}" +} + +# Build up the full list of search directories +declare -a needed_search_path +do_trace "Adding basic lib dirs\n" +ld_library_path="${ld_library_path}:" +while [ -n "${ld_library_path}" ]; do + d="${ld_library_path%%:*}" + if [ -n "${d}" ]; then + do_trace "-> adding search dir '%s'\n" "${d}" + needed_search_path+=( "${d}" ) + fi + ld_library_path="${ld_library_path#*:}" +done +do_trace "Done adding basic lib dirs\n" +do_trace "Scanning '/etc/ld.so.conf'\n" +do_scan_etc_ldsoconf "${root}/etc/ld.so.conf" +do_trace "Done scanning '/etc/ld.so.conf'\n" +do_trace "Search path:\n" +for p in "${needed_search_path[@]}"; do + do_trace "-> '%s'\n" "${p}" +done + +declare -a needed_list +declare -a search_rpath +do_trace "Scanning file '%s'\n" "${1}" +do_process_file "${1}" +do_trace "Done scanning file '%s'\n" "${1}" diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index f5bedbd3..b9df1ccb 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -1,11 +1,13 @@ python: - packages: + packages_trusty: apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' + packages_stretch: + apt: 'make build-essential libssl1.0-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' brew: setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox"' env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"' nodejs: - packages: + packages_trusty: apt: 'nodejs sox' prep_4: 'echo "deb http://deb.nodesource.com/node_4.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_5: 'echo "deb http://deb.nodesource.com/node_5.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' @@ -13,6 +15,14 @@ nodejs: prep_7: 'echo "deb http://deb.nodesource.com/node_7.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_8: 'echo "deb http://deb.nodesource.com/node_8.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_9: 'echo "deb http://deb.nodesource.com/node_9.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + packages_stretch: + apt: 'nodejs sox' + prep_4: 'echo "deb http://deb.nodesource.com/node_4.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_5: 'echo "deb http://deb.nodesource.com/node_5.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_6: 'echo "deb http://deb.nodesource.com/node_6.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_7: 'echo "deb http://deb.nodesource.com/node_7.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_8: 'echo "deb http://deb.nodesource.com/node_8.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_9: 'echo "deb http://deb.nodesource.com/node_9.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' brew: # Using 'nvm' from homebrew results in installing v0.33.6 which seems to # suffer from a bug making it unable to work as of now: @@ -32,7 +42,7 @@ nodejs: ## force 3.0.10 for nodejs V8 API compatibility swig: packages: - install_script: 'wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig_3.0.10-1.2_amd64.deb -O /tmp/swig_3.0.10-1.2_amd64.deb && wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig3.0_3.0.10-1.2_amd64.deb -O /tmp/swig3.0_3.0.10-1.2_amd64.deb && dpkg -i /tmp/swig_3.0.10-1.2_amd64.deb /tmp/swig3.0_3.0.10-1.2_amd64.deb' + install_script: 'wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig_3.0.12-1_amd64.deb -O /tmp/swig_3.0.12-1_amd64.deb && wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig3.0_3.0.12-1_amd64.deb -O /tmp/swig3.0_3.0.12-1_amd64.deb && dpkg -i /tmp/swig_3.0.12-1_amd64.deb /tmp/swig3.0_3.0.12-1_amd64.deb' patch_nodejs: linux: '(for patch_file in /home/build-user/DeepSpeech/ds/native_client/swig_node_v7x-v8x-v9x_*.patch; do patch -d /usr/share/swig3.0/ -p2 < $patch_file; done)' # Test if we can reverse patch #2: @@ -62,4 +72,4 @@ system: osx: '/Users/build-user' notifications: irc: '#machinelearning' -aptEc2Mirrors: 'sed -ri -e "s|archive.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" -e "s|security.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" /etc/apt/sources.list' +aptEc2Mirrors: 'echo "deb http://archive.ubuntu.com/ubuntu/ trusty-updates main" > /etc/apt/sources.list.d/trusty-updates.list && sed -ri -e "s|archive.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" -e "s|security.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" /etc/apt/sources.list && apt-get -qq update && apt-get -qq -y upgrade' diff --git a/taskcluster/linux-amd64-cpu-aot_prod-opt.yml b/taskcluster/linux-amd64-cpu-aot_prod-opt.yml index a80291d4..da330aaa 100644 --- a/taskcluster/linux-amd64-cpu-aot_prod-opt.yml +++ b/taskcluster/linux-amd64-cpu-aot_prod-opt.yml @@ -8,8 +8,8 @@ build: summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.4d842daffdfde9da80d14fea3e3befeeb994872a.cpu/artifacts/public/summarize_graph" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > ${swig.patch_nodejs.linux} diff --git a/taskcluster/linux-amd64-cpu-aot_test-opt.yml b/taskcluster/linux-amd64-cpu-aot_test-opt.yml index 549c3be2..cb2c029c 100644 --- a/taskcluster/linux-amd64-cpu-aot_test-opt.yml +++ b/taskcluster/linux-amd64-cpu-aot_test-opt.yml @@ -10,8 +10,8 @@ build: summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.4d842daffdfde9da80d14fea3e3befeeb994872a.cpu/artifacts/public/summarize_graph" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > ${swig.patch_nodejs.linux} diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 5b13ede7..293a6929 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -8,8 +8,8 @@ build: - "notify.irc-channel.${notifications.irc}.on-failed" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > ${swig.patch_nodejs.linux} diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 4e7b0fc1..3ea3847b 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -6,8 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > ${swig.patch_nodejs.linux} diff --git a/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml b/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml index 6fa4a63b..e9663e77 100644 --- a/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml +++ b/taskcluster/linux-rpi3-cpu-aot_prod-opt.yml @@ -4,16 +4,16 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm_aot" - "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm_aot" - "index.project.deepspeech.deepspeech.native_client.arm_aot.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.4d842daffdfde9da80d14fea3e3befeeb994872a.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.4d842daffdfde9da80d14fea3e3befeeb994872a.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.6.236f83eb5d4d73a33938154f4b1e631355f6a1f0.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.6.236f83eb5d4d73a33938154f4b1e631355f6a1f0.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > multistrap -d /tmp/multistrap-raspbian-jessie/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap.conf && @@ -22,5 +22,5 @@ build: build: "taskcluster/rpi3-build.sh --aot" package: "taskcluster/package.sh" metadata: - name: "DeepSpeech Linux RPi3/ARMv6 CPU AOT" - description: "Building DeepSpeech for Linux RPi3 ARMv6, AOT model, CPU only, optimized version" + name: "DeepSpeech Linux RPi3/ARMv7 CPU AOT" + description: "Building DeepSpeech for Linux RPi3 ARMv7, AOT model, CPU only, optimized version" diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 2895e197..d4562889 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -12,8 +12,8 @@ build: apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > multistrap -d /tmp/multistrap-raspbian-jessie/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap.conf && @@ -22,5 +22,5 @@ build: build: "taskcluster/rpi3-build.sh" package: "taskcluster/package.sh" metadata: - name: "DeepSpeech Linux RPi3/ARMv6 CPU" - description: "Building DeepSpeech for Linux RPi3 ARMv6, CPU only, optimized version" + name: "DeepSpeech Linux RPi3/ARMv7 CPU" + description: "Building DeepSpeech for Linux RPi3 ARMv7, CPU only, optimized version" diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index 0d690292..448f0d81 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -9,8 +9,8 @@ build: - "notify.irc-channel.${notifications.irc}.on-failed" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: > ${swig.patch_nodejs.linux} diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh index 3b57f3d8..d5ef47a4 100755 --- a/taskcluster/rpi3-build.sh +++ b/taskcluster/rpi3-build.sh @@ -27,11 +27,13 @@ if [ $1 = "--aot" ]; then BAZEL_BUILD_FLAGS="${BAZEL_BUILD_FLAGS} ${BAZEL_AOT_BUILD_FLAGS} ${AOT_MODEL_PARAMS}" fi; +maybe_install_xldd + do_bazel_build do_deepspeech_binary_build -export SUPPORTED_PYTHON_VERSIONS="2.7.14:ucs4 3.4.8:ucs4" +export SUPPORTED_PYTHON_VERSIONS="3.4.8:ucs4 3.5.3:ucs4" do_deepspeech_python_build do_deepspeech_nodejs_build diff --git a/taskcluster/test-benchmark-linux-amd64-aot_test-opt.yml b/taskcluster/test-benchmark-linux-amd64-aot_test-opt.yml index 7d254c99..c545488f 100644 --- a/taskcluster/test-benchmark-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-benchmark-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} zip + apt-get -qq -y install ${python.packages_trusty.apt} zip args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-benchmark-tests.sh --aot" metadata: diff --git a/taskcluster/test-benchmark-linux-amd64-opt.yml b/taskcluster/test-benchmark-linux-amd64-opt.yml index 5af0a34a..3b27e8ff 100644 --- a/taskcluster/test-benchmark-linux-amd64-opt.yml +++ b/taskcluster/test-benchmark-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} zip + apt-get -qq -y install ${python.packages_trusty.apt} zip args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-benchmark-tests.sh" metadata: diff --git a/taskcluster/test-cpp-raspbian-rpi3-aot_test-opt.yml b/taskcluster/test-cpp-raspbian-rpi3-aot_test-opt.yml new file mode 100644 index 00000000..c09d7e78 --- /dev/null +++ b/taskcluster/test-cpp-raspbian-rpi3-aot_test-opt.yml @@ -0,0 +1,10 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-aot_prod-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-cpp-ds-tests.sh --aot" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU C++ tests AOT (Test)" + description: "Testing DeepSpeech C++ for Raspbian RPi3/ARMv7, AOT Model (test), CPU only, optimized version" diff --git a/taskcluster/test-cpp-raspbian-rpi3-opt.yml b/taskcluster/test-cpp-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..b2a1c5bb --- /dev/null +++ b/taskcluster/test-cpp-raspbian-rpi3-opt.yml @@ -0,0 +1,10 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-cpp-ds-tests.sh" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU C++ tests" + description: "Testing DeepSpeech C++ for Raspbian RPi3/ARMv7, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_4x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_4x-linux-amd64-aot_test-opt.yml index 2940c5a3..fdfa8d02 100644 --- a/taskcluster/test-nodejs_4x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_4x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 4.x --aot" metadata: diff --git a/taskcluster/test-nodejs_4x-linux-amd64-opt.yml b/taskcluster/test-nodejs_4x-linux-amd64-opt.yml index 72e4aafc..54b1a409 100644 --- a/taskcluster/test-nodejs_4x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_4x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 4.x" metadata: diff --git a/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml index ecb80181..e75aa77f 100644 --- a/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 4.x" metadata: diff --git a/taskcluster/test-nodejs_4x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_4x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..069ed838 --- /dev/null +++ b/taskcluster/test-nodejs_4x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_4} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 4.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 4.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v4.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_5x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_5x-linux-amd64-aot_test-opt.yml index b7d986a8..d2ad2389 100644 --- a/taskcluster/test-nodejs_5x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_5x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 5.x --aot" metadata: diff --git a/taskcluster/test-nodejs_5x-linux-amd64-opt.yml b/taskcluster/test-nodejs_5x-linux-amd64-opt.yml index 7382eb00..b0d9a323 100644 --- a/taskcluster/test-nodejs_5x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_5x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 5.x" metadata: diff --git a/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml index 6ffcd118..270ccc59 100644 --- a/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 5.x" metadata: diff --git a/taskcluster/test-nodejs_5x-raspbian-rpi3-opt.yml-disabled b/taskcluster/test-nodejs_5x-raspbian-rpi3-opt.yml-disabled new file mode 100644 index 00000000..5a1c1aa0 --- /dev/null +++ b/taskcluster/test-nodejs_5x-raspbian-rpi3-opt.yml-disabled @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_5} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 5.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 5.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v5.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_6x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_6x-linux-amd64-aot_test-opt.yml index 2aba2b81..640478f2 100644 --- a/taskcluster/test-nodejs_6x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_6x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 6.x --aot" metadata: diff --git a/taskcluster/test-nodejs_6x-linux-amd64-opt.yml b/taskcluster/test-nodejs_6x-linux-amd64-opt.yml index 1559b5d6..419baddf 100644 --- a/taskcluster/test-nodejs_6x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_6x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 6.x" metadata: diff --git a/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml index 879a0286..6837c60b 100644 --- a/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 6.x" metadata: diff --git a/taskcluster/test-nodejs_6x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_6x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..90107045 --- /dev/null +++ b/taskcluster/test-nodejs_6x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 6.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 6.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v6.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_7x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_7x-linux-amd64-aot_test-opt.yml index 015f3baf..6b1d8f1f 100644 --- a/taskcluster/test-nodejs_7x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_7x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 7.x --aot" metadata: diff --git a/taskcluster/test-nodejs_7x-linux-amd64-opt.yml b/taskcluster/test-nodejs_7x-linux-amd64-opt.yml index 1fac617b..768d3061 100644 --- a/taskcluster/test-nodejs_7x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_7x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 7.x" metadata: diff --git a/taskcluster/test-nodejs_7x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_7x-linux-amd64-prod_pbmodel-opt.yml index 8d579cb7..4e45203b 100644 --- a/taskcluster/test-nodejs_7x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_7x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 7.x" metadata: diff --git a/taskcluster/test-nodejs_7x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_7x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..6f9bff22 --- /dev/null +++ b/taskcluster/test-nodejs_7x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_7} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 7.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 7.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v7.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_8x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_8x-linux-amd64-aot_test-opt.yml index 1dac03e5..d8020a25 100644 --- a/taskcluster/test-nodejs_8x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_8x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 8.x --aot" metadata: diff --git a/taskcluster/test-nodejs_8x-linux-amd64-opt.yml b/taskcluster/test-nodejs_8x-linux-amd64-opt.yml index dffef704..9233e420 100644 --- a/taskcluster/test-nodejs_8x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_8x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 8.x" metadata: diff --git a/taskcluster/test-nodejs_8x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_8x-linux-amd64-prod_pbmodel-opt.yml index e09d66b4..300839ff 100644 --- a/taskcluster/test-nodejs_8x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_8x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 8.x" metadata: diff --git a/taskcluster/test-nodejs_8x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_8x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..f18c9b17 --- /dev/null +++ b/taskcluster/test-nodejs_8x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_8} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 8.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 8.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v8.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_9x-linux-amd64-aot_test-opt.yml b/taskcluster/test-nodejs_9x-linux-amd64-aot_test-opt.yml index 49838480..4d364d8a 100644 --- a/taskcluster/test-nodejs_9x-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-nodejs_9x-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 9.x --aot" metadata: diff --git a/taskcluster/test-nodejs_9x-linux-amd64-opt.yml b/taskcluster/test-nodejs_9x-linux-amd64-opt.yml index 0638dfaf..c6553536 100644 --- a/taskcluster/test-nodejs_9x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_9x-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 9.x" metadata: diff --git a/taskcluster/test-nodejs_9x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_9x-linux-amd64-prod_pbmodel-opt.yml index 0823417c..d5c96153 100644 --- a/taskcluster/test-nodejs_9x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_9x-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt} + ${nodejs.packages_trusty.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 9.x" metadata: diff --git a/taskcluster/test-nodejs_9x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_9x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..6a6800d8 --- /dev/null +++ b/taskcluster/test-nodejs_9x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_9} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 9.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 9.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v9.x, CPU only, optimized version" diff --git a/taskcluster/test-python_27m-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_27m-linux-amd64-aot_test-opt.yml index c65e864f..55915a58 100644 --- a/taskcluster/test-python_27m-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-python_27m-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 2.7.14:m --aot" metadata: diff --git a/taskcluster/test-python_27m-linux-amd64-opt.yml b/taskcluster/test-python_27m-linux-amd64-opt.yml index 94a14091..ee3ee7a4 100644 --- a/taskcluster/test-python_27m-linux-amd64-opt.yml +++ b/taskcluster/test-python_27m-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 2.7.14:m" metadata: diff --git a/taskcluster/test-python_27m-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_27m-linux-amd64-prod_pbmodel-opt.yml index 46f984b3..9c3130b3 100644 --- a/taskcluster/test-python_27m-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_27m-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 2.7.14:m" metadata: diff --git a/taskcluster/test-python_27mu-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_27mu-linux-amd64-aot_test-opt.yml index 3a7f9fed..51c71b51 100644 --- a/taskcluster/test-python_27mu-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-python_27mu-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 2.7.14:mu --aot" metadata: diff --git a/taskcluster/test-python_27mu-linux-amd64-opt.yml b/taskcluster/test-python_27mu-linux-amd64-opt.yml index dd31d486..03632ce5 100644 --- a/taskcluster/test-python_27mu-linux-amd64-opt.yml +++ b/taskcluster/test-python_27mu-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 2.7.14:mu" metadata: diff --git a/taskcluster/test-python_27mu-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_27mu-linux-amd64-prod_pbmodel-opt.yml index d3d8ed8a..e475abcb 100644 --- a/taskcluster/test-python_27mu-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_27mu-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 2.7.14:mu" metadata: diff --git a/taskcluster/test-python_34m-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_34m-linux-amd64-aot_test-opt.yml index 048dd134..29e47abb 100644 --- a/taskcluster/test-python_34m-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-python_34m-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.4.8:m --aot" metadata: diff --git a/taskcluster/test-python_34m-linux-amd64-opt.yml b/taskcluster/test-python_34m-linux-amd64-opt.yml index 8d2b6824..b0e88518 100644 --- a/taskcluster/test-python_34m-linux-amd64-opt.yml +++ b/taskcluster/test-python_34m-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.4.8:m" metadata: diff --git a/taskcluster/test-python_34m-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_34m-linux-amd64-prod_pbmodel-opt.yml index 2f253d70..37528282 100644 --- a/taskcluster/test-python_34m-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_34m-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 3.4.8:m" metadata: diff --git a/taskcluster/test-python_34m-raspbian-rpi3-opt.yml b/taskcluster/test-python_34m-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..ff5a0f93 --- /dev/null +++ b/taskcluster/test-python_34m-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.4.8:m" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.4 tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.4, CPU only, optimized version" diff --git a/taskcluster/test-python_35-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_35-linux-amd64-aot_test-opt.yml index d4280a6a..9667d0c2 100644 --- a/taskcluster/test-python_35-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-python_35-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.5.5:m --aot" metadata: diff --git a/taskcluster/test-python_35-linux-amd64-opt.yml b/taskcluster/test-python_35-linux-amd64-opt.yml index a3932a2a..e55eb3f7 100644 --- a/taskcluster/test-python_35-linux-amd64-opt.yml +++ b/taskcluster/test-python_35-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.5.5:m" metadata: diff --git a/taskcluster/test-python_35-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_35-linux-amd64-prod_pbmodel-opt.yml index a52d3387..92330e6f 100644 --- a/taskcluster/test-python_35-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_35-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 3.5.5:m" metadata: diff --git a/taskcluster/test-python_35m-raspbian-rpi3-opt.yml b/taskcluster/test-python_35m-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..5d7d2922 --- /dev/null +++ b/taskcluster/test-python_35m-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.5.5:m" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU Python v3.5 tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on Python v3.5, CPU only, optimized version" diff --git a/taskcluster/test-python_36-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_36-linux-amd64-aot_test-opt.yml index 3b02cc8b..76a309e6 100644 --- a/taskcluster/test-python_36-linux-amd64-aot_test-opt.yml +++ b/taskcluster/test-python_36-linux-amd64-aot_test-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.6.4:m --aot" metadata: diff --git a/taskcluster/test-python_36-linux-amd64-opt.yml b/taskcluster/test-python_36-linux-amd64-opt.yml index 11fb5b37..9b723ae9 100644 --- a/taskcluster/test-python_36-linux-amd64-opt.yml +++ b/taskcluster/test-python_36-linux-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.6.4:m" metadata: diff --git a/taskcluster/test-python_36-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_36-linux-amd64-prod_pbmodel-opt.yml index 5e55267a..4400848e 100644 --- a/taskcluster/test-python_36-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_36-linux-amd64-prod_pbmodel-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 3.6.4:m" metadata: diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml new file mode 100644 index 00000000..c348710c --- /dev/null +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -0,0 +1,76 @@ +$if: 'event.event != "push"' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.dockerrpi3.provisionerId} + workerType: ${taskcluster.dockerrpi3.workerType} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '7 days' } + + extra: + github: + { $eval: taskcluster.github_events.pull_request } + + routes: + - "notify.irc-channel.${notifications.irc}.on-exception" + - "notify.irc-channel.${notifications.irc}.on-failed" + + scopes: [ + "queue:route:notify.irc-channel.*" + ] + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: "resin/rpi-raspbian:stretch-20180228" + + env: + $let: + training: { $eval: as_slugid("test-training_upstream-linux-amd64-py27mu-opt") } + linux_rpi3_aot_prod: { $eval: as_slugid("linux-rpi3-cpu-aot_prod-opt") } + linux_rpi3_build: { $eval: as_slugid("linux-rpi3-cpu-opt") } + node_package: { $eval: as_slugid("node-package") } + in: + CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef} + DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_rpi3_build}/artifacts/public + DEEPSPEECH_PYTHON_PACKAGE: https://queue.taskcluster.net/v1/task/${linux_rpi3_build}/artifacts/public/${build.deepspeech_pkg_name} + DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public + DEEPSPEECH_AOT_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_rpi3_aot_prod}/artifacts/public + DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb + DEEPSPEECH_PROD_MODEL: https://s3.amazonaws.com/deep-speech/mmap/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://s3.amazonaws.com/deep-speech/mmap/output_graph.pbmm + PIP_DEFAULT_TIMEOUT: 60 + PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" + EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.6.0-14-ge4869c8" + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + in: > + apt-get -qq update && apt-get -qq -y install git pixz sox wget libatlas3-base && ${extraSystemSetup} && + adduser --system --home ${system.homedir.linux} ${system.username} && + cd ${system.homedir.linux} && + echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline} + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '7 days' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py27mu-opt.yml-disabled b/taskcluster/test-training+deepspeech-linux-amd64-py27mu-opt.yml-disabled index ac1b5468..87a758a2 100644 --- a/taskcluster/test-training+deepspeech-linux-amd64-py27mu-opt.yml-disabled +++ b/taskcluster/test-training+deepspeech-linux-amd64-py27mu-opt.yml-disabled @@ -5,7 +5,7 @@ build: - "linux-amd64-cpu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu mozilla deepspeech" deepspeech_pkg_name: 'deepspeech-0.1.1-cp27-cp27mu-manylinux1_x86_64.whl' diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py36m-opt.yml-disabled b/taskcluster/test-training+deepspeech-linux-amd64-py36m-opt.yml-disabled index 787bddc2..993601dc 100644 --- a/taskcluster/test-training+deepspeech-linux-amd64-py36m-opt.yml-disabled +++ b/taskcluster/test-training+deepspeech-linux-amd64-py36m-opt.yml-disabled @@ -5,7 +5,7 @@ build: - "linux-amd64-cpu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m mozilla deepspeech" deepspeech_pkg_name: 'deepspeech-0.1.1-cp36-cp36m-manylinux1_x86_64.whl' diff --git a/taskcluster/test-training-linux-amd64-py27mu-opt.yml-disabled b/taskcluster/test-training-linux-amd64-py27mu-opt.yml-disabled index e234c44d..18681640 100644 --- a/taskcluster/test-training-linux-amd64-py27mu-opt.yml-disabled +++ b/taskcluster/test-training-linux-amd64-py27mu-opt.yml-disabled @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu mozilla" metadata: diff --git a/taskcluster/test-training-linux-amd64-py34m-opt.yml-disabled b/taskcluster/test-training-linux-amd64-py34m-opt.yml-disabled index 085b2f8d..8a83077f 100644 --- a/taskcluster/test-training-linux-amd64-py34m-opt.yml-disabled +++ b/taskcluster/test-training-linux-amd64-py34m-opt.yml-disabled @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.4.8:m mozilla" metadata: diff --git a/taskcluster/test-training-linux-amd64-py35m-opt.yml-disabled b/taskcluster/test-training-linux-amd64-py35m-opt.yml-disabled index a110f7d5..adc52709 100644 --- a/taskcluster/test-training-linux-amd64-py35m-opt.yml-disabled +++ b/taskcluster/test-training-linux-amd64-py35m-opt.yml-disabled @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.5.5:m mozilla" metadata: diff --git a/taskcluster/test-training-linux-amd64-py36m-opt.yml-disabled b/taskcluster/test-training-linux-amd64-py36m-opt.yml-disabled index 64a452b5..5153a6a7 100644 --- a/taskcluster/test-training-linux-amd64-py36m-opt.yml-disabled +++ b/taskcluster/test-training-linux-amd64-py36m-opt.yml-disabled @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m mozilla" metadata: diff --git a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27mu-opt.yml index 2a64af6b..97ab81b9 100644 --- a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27mu-opt.yml @@ -5,7 +5,7 @@ build: - "linux-amd64-cpu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu upstream deepspeech" deepspeech_pkg_name: 'deepspeech-0.1.1-cp27-cp27mu-manylinux1_x86_64.whl' diff --git a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36m-opt.yml b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36m-opt.yml index 74043eaf..a797e462 100644 --- a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36m-opt.yml @@ -5,7 +5,7 @@ build: - "linux-amd64-cpu-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m upstream deepspeech" deepspeech_pkg_name: 'deepspeech-0.1.1-cp36-cp36m-manylinux1_x86_64.whl' diff --git a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml index 82773c82..027ef26c 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu upstream" convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.4d842daffdfde9da80d14fea3e3befeeb994872a.cpu/artifacts/public/convert_graphdef_memmapped_format" diff --git a/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml index f20e52c1..154c2846 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py34m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.4.8:m upstream" metadata: diff --git a/taskcluster/test-training_upstream-linux-amd64-py35m-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py35m-opt.yml index a6fd2913..19636f54 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py35m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.5.5:m upstream" metadata: diff --git a/taskcluster/test-training_upstream-linux-amd64-py36m-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py36m-opt.yml index 0dca9122..6f39cf16 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq -y install ${python.packages.apt} + apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m upstream" metadata: diff --git a/taskcluster/worker.cyml b/taskcluster/worker.cyml index 0293f88f..4ac1dd8f 100644 --- a/taskcluster/worker.cyml +++ b/taskcluster/worker.cyml @@ -3,6 +3,9 @@ taskcluster: docker: provisionerId: aws-provisioner-v1 workerType: deepspeech-worker + dockerrpi3: + provisionerId: deepspeech-provisioner + workerType: ds-rpi3 generic: provisionerId: deepspeech-provisioner workerType: ds-macos-light diff --git a/tc-python-tests-prod.sh b/tc-python-tests-prod.sh index e7069a4c..4763c514 100644 --- a/tc-python-tests-prod.sh +++ b/tc-python-tests-prod.sh @@ -42,14 +42,14 @@ install_pyenv "${PYENV_ROOT}" install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" PYENV_NAME=deepspeech-test -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver} +PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver} pyenv virtualenv ${pyver} ${PYENV_NAME} source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); plat = "manylinux1" if plat == "linux" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') +platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') deepspeech_pkg="deepspeech-0.1.1-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" -pip install --upgrade ${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} | cat +pip install --only-binary :all: --upgrade ${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} | cat run_prod_inference_tests diff --git a/tc-python-tests.sh b/tc-python-tests.sh index 12c2ce56..9b3ed376 100644 --- a/tc-python-tests.sh +++ b/tc-python-tests.sh @@ -37,11 +37,11 @@ install_pyenv "${PYENV_ROOT}" install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" PYENV_NAME=deepspeech-test -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver} +PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver} pyenv virtualenv ${pyver} ${PYENV_NAME} source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); plat = "manylinux1" if plat == "linux" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') +platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') deepspeech_pkg="deepspeech-0.1.1-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" if [ "${aot_model}" = "--aot" ]; then @@ -49,7 +49,7 @@ if [ "${aot_model}" = "--aot" ]; then else deepspeech_pkg_url=${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} fi -pip install --upgrade ${deepspeech_pkg_url} | cat +pip install --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat run_all_inference_tests diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index e3b9d4c5..9b1ab8ca 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -43,7 +43,7 @@ model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm" model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}" SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.14:ucs2 2.7.14:ucs4 3.4.8:ucs4 3.5.5:ucs4 3.6.4:ucs4} -SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.8.6 5.12.0 6.12.0 7.10.1 8.9.1 9.2.0} +SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.14.1 7.10.1 8.11.1 9.11.1} # This verify exact inference result assert_correct_inference() @@ -367,6 +367,15 @@ install_pyenv_virtualenv() eval "$(pyenv virtualenv-init -)" } +maybe_install_xldd() +{ + # -s required to avoid the noisy output like "Entering / Leaving directories" + toolchain=$(make -s -C ${DS_DSDIR}/native_client/ TARGET=${SYSTEM_TARGET} TFDIR=${DS_TFDIR} print-toolchain) + if [ ! -x "${toolchain}ldd" ]; then + cp "${DS_DSDIR}/native_client/xldd" "${toolchain}ldd" && chmod +x "${toolchain}ldd" + fi +} + do_get_model_parameters() { local __result=$2 @@ -415,6 +424,10 @@ verify_bazel_rebuild() exit 1 fi; + mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + + cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ + spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested' | wc -l) if [ "${spurious_rebuilds}" -ne 0 ]; then echo "Bazel rebuilds some file it should not, please check."