From 968c22c392bf1a6d305542e72bfa2d5ec5bb6113 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 19 Dec 2017 05:03:12 +0100 Subject: [PATCH] Avoid over-linking deepspeech.utils Python wrapper Fixes #1116 --- native_client/Makefile | 2 +- native_client/definitions.mk | 16 +++++++++++----- native_client/setup.py | 4 +++- taskcluster/.build.yml | 1 + taskcluster/test-linux-opt-base.tyml | 1 + ...-training+deepspeech-linux-amd64-py27-opt.yml | 15 +++++++++++++++ ...-training+deepspeech-linux-amd64-py36-opt.yml | 15 +++++++++++++++ ..._upstream+deepspeech-linux-amd64-py27-opt.yml | 14 ++++++++++++++ ..._upstream+deepspeech-linux-amd64-py36-opt.yml | 14 ++++++++++++++ tc-train-tests.sh | 6 ++++++ 10 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml create mode 100644 taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml create mode 100644 taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml create mode 100644 taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml diff --git a/native_client/Makefile b/native_client/Makefile index a6aaae74..8dda7729 100644 --- a/native_client/Makefile +++ b/native_client/Makefile @@ -32,7 +32,7 @@ bindings-clean: bindings-build: pip install --quiet $(PYTHON_PACKAGES) wheel - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(RPATH_PYTHON) $(LDFLAGS)" LIBS="$(LIBS)" $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" UTILS_LDFLAGS="-L${TFDIR}/bazel-bin/native_client" MODEL_LIBS="$(LIBS)" $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ diff --git a/native_client/definitions.mk b/native_client/definitions.mk index c5ea6488..85c7b2f9 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -38,13 +38,19 @@ OS := $(shell uname -s) # -Wl,--no-as-needed is required to force linker not to evict libs it thinks we # dont need ; will fail the build on OSX because that option does not exists -ifneq ($(OS),Darwin) -LDFLAGS += -Wl,--no-as-needed +ifeq ($(OS),Linux) +LDFLAGS_NEEDED := -Wl,--no-as-needed +LDFLAGS_RPATH := -Wl,-rpath,\$$ORIGIN +endif +ifeq ($(OS),Darwin) +LDFLAGS_NEEDED := +LDFLAGS_RPATH := -Wl,-rpath,@executable_path endif CFLAGS += $(EXTRA_CFLAGS) LIBS := -ldeepspeech -ldeepspeech_utils -ltensorflow_cc -ltensorflow_framework $(EXTRA_LIBS) -LDFLAGS += -Wl,-rpath,. -L${TFDIR}/bazel-bin/tensorflow -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) $(LIBS) +LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/tensorflow -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) +LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS) AS := $(TOOLCHAIN)as CC := $(TOOLCHAIN)gcc @@ -52,12 +58,12 @@ CXX := $(TOOLCHAIN)c++ LD := $(TOOLCHAIN)ld LDD := $(TOOLCHAIN)ldd $(TOOLCHAIN_LDD_OPTS) -RPATH_PYTHON := '-Wl,-rpath,\$$ORIGIN/lib/' +RPATH_PYTHON := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH) RPATH_NODEJS := '-Wl,-rpath,$$\$$ORIGIN/../' META_LD_LIBRARY_PATH := LD_LIBRARY_PATH ifeq ($(OS),Darwin) META_LD_LIBRARY_PATH := DYLD_LIBRARY_PATH -RPATH_PYTHON := '-Wl,-rpath,@loader_path/lib/' +RPATH_PYTHON := '-Wl,-rpath,@loader_path/lib/' $(LDFLAGS_RPATH) RPATH_NODEJS := '-Wl,-rpath,@loader_path/../' endif diff --git a/native_client/setup.py b/native_client/setup.py index 9cc74405..22999822 100755 --- a/native_client/setup.py +++ b/native_client/setup.py @@ -35,11 +35,13 @@ class BuildExtFirst(build): model = Extension('deepspeech._model', ['python/model.i'], include_dirs = [numpy_include], - libraries = list(map(lambda x: x.strip(), os.getenv('LIBS', '').split('-l')[1:]))) + library_dirs = list(map(lambda x: x.strip(), os.getenv('MODEL_LDFLAGS', '').split('-L')[1:])), + libraries = list(map(lambda x: x.strip(), os.getenv('MODEL_LIBS', '').split('-l')[1:]))) utils = Extension('deepspeech._utils', ['python/utils.i'], include_dirs = [numpy_include], + library_dirs = list(map(lambda x: x.strip(), os.getenv('UTILS_LDFLAGS', '').split('-L')[1:])), libraries = ['deepspeech_utils']) setup(name = project_name, diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 2e35118b..febb384e 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -30,3 +30,4 @@ build: args: tests_cmdline: '' python_wheel: '' + deepspeech_pkg_name: '' diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 930ef0f1..005a2455 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -39,6 +39,7 @@ then: in: TENSORFLOW_WHEEL: ${build.python_wheel} DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public + DEEPSPEECH_PYTHON_PACKAGE: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public/${build.deepspeech_pkg_name} DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public DEEPSPEECH_AOT_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_aot_test}/artifacts/public DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml b/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml new file mode 100644 index 00000000..72415263 --- /dev/null +++ b/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 mozilla deepspeech" + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl' + deepspeech_pkg_name: 'deepspeech-0.1.0-cp27-cp27mu-manylinux1_x86_64.whl' + metadata: + name: "DeepSpeech Linux AMD64 CPU training Py2.7 (DS)" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 2.7 with DeepSpeech package installed, CPU only, optimized version" diff --git a/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml b/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml new file mode 100644 index 00000000..db37b2c1 --- /dev/null +++ b/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 mozilla deepspeech" + python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl' + deepspeech_pkg_name: 'deepspeech-0.1.0-cp36-cp36m-manylinux1_x86_64.whl' + metadata: + name: "DeepSpeech Linux AMD64 CPU training Py3.6 (DS)" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6 with DeepSpeech package installed, CPU only, optimized version" diff --git a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml new file mode 100644 index 00000000..75dbde67 --- /dev/null +++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 upstream deepspeech" + deepspeech_pkg_name: 'deepspeech-0.1.0-cp27-cp27mu-manylinux1_x86_64.whl' + metadata: + name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 (DS)" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 with DeepSpeech package installed, CPU only, optimized version" diff --git a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml new file mode 100644 index 00000000..cfe78756 --- /dev/null +++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 upstream deepspeech" + deepspeech_pkg_name: 'deepspeech-0.1.0-cp36-cp36m-manylinux1_x86_64.whl' + metadata: + name: "DeepSpeech Linux AMD64 CPU upstream training Py3.6 (DS)" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 3.6 with DeepSpeech package installed, CPU only, optimized version" diff --git a/tc-train-tests.sh b/tc-train-tests.sh index 572b5b1d..677fb7dc 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -6,6 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh pyver=$1 tf=$2 +ds=$3 if [ -z "${pyver}" ]; then echo "No python version given, aborting." @@ -38,6 +39,11 @@ if [ "${tf}" = "upstream" ]; then pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt fi; +if [ "${ds}" = "deepspeech" ]; then + pip install "${DEEPSPEECH_PYTHON_PACKAGE}" + python -c "import tensorflow; from deepspeech.utils import audioToInputVector" +fi; + download_ctc_kenlm "/tmp/ds" pushd ${HOME}/DeepSpeech/ds/