Avoid over-linking deepspeech.utils Python wrapper

Fixes #1116
2025-10-26 11:19:39 +00:00 · 2017-12-19 05:03:12 +01:00 · 2017-12-19 05:03:12 +01:00 · 968c22c392
commit 968c22c392
parent cbc868346b
10 changed files with 81 additions and 7 deletions
--- a/native_client/Makefile
+++ b/native_client/Makefile
@ -32,7 +32,7 @@ bindings-clean:

 bindings-build:
 	pip install --quiet $(PYTHON_PACKAGES) wheel
-	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(RPATH_PYTHON) $(LDFLAGS)" LIBS="$(LIBS)" $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
+	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" UTILS_LDFLAGS="-L${TFDIR}/bazel-bin/native_client" MODEL_LIBS="$(LIBS)" $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)

 MANIFEST.in: bindings-build
 	> $@
--- a/native_client/definitions.mk
+++ b/native_client/definitions.mk
@ -38,13 +38,19 @@ OS      := $(shell uname -s)

 # -Wl,--no-as-needed is required to force linker not to evict libs it thinks we
 # dont need ; will fail the build on OSX because that option does not exists
-ifneq ($(OS),Darwin)
-LDFLAGS += -Wl,--no-as-needed
+ifeq ($(OS),Linux)
+LDFLAGS_NEEDED := -Wl,--no-as-needed
+LDFLAGS_RPATH  := -Wl,-rpath,\$$ORIGIN
+endif
+ifeq ($(OS),Darwin)
+LDFLAGS_NEEDED :=
+LDFLAGS_RPATH  := -Wl,-rpath,@executable_path
 endif

 CFLAGS  += $(EXTRA_CFLAGS)
 LIBS    := -ldeepspeech -ldeepspeech_utils -ltensorflow_cc -ltensorflow_framework $(EXTRA_LIBS)
-LDFLAGS += -Wl,-rpath,. -L${TFDIR}/bazel-bin/tensorflow -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS) $(LIBS)
+LDFLAGS_DIRS := -L${TFDIR}/bazel-bin/tensorflow -L${TFDIR}/bazel-bin/native_client $(EXTRA_LDFLAGS)
+LDFLAGS += $(LDFLAGS_NEEDED) $(LDFLAGS_RPATH) $(LDFLAGS_DIRS) $(LIBS)

 AS      := $(TOOLCHAIN)as
 CC      := $(TOOLCHAIN)gcc
@ -52,12 +58,12 @@ CXX     := $(TOOLCHAIN)c++
 LD      := $(TOOLCHAIN)ld
 LDD     := $(TOOLCHAIN)ldd $(TOOLCHAIN_LDD_OPTS)

-RPATH_PYTHON         := '-Wl,-rpath,\$$ORIGIN/lib/'
+RPATH_PYTHON         := '-Wl,-rpath,\$$ORIGIN/lib/' $(LDFLAGS_RPATH)
 RPATH_NODEJS         := '-Wl,-rpath,$$\$$ORIGIN/../'
 META_LD_LIBRARY_PATH := LD_LIBRARY_PATH
 ifeq ($(OS),Darwin)
 META_LD_LIBRARY_PATH := DYLD_LIBRARY_PATH
-RPATH_PYTHON         := '-Wl,-rpath,@loader_path/lib/'
+RPATH_PYTHON         := '-Wl,-rpath,@loader_path/lib/' $(LDFLAGS_RPATH)
 RPATH_NODEJS         := '-Wl,-rpath,@loader_path/../'
 endif

--- a/native_client/setup.py
+++ b/native_client/setup.py
@ -35,11 +35,13 @@ class BuildExtFirst(build):
 model = Extension('deepspeech._model',
        ['python/model.i'],
        include_dirs = [numpy_include],
-        libraries = list(map(lambda x: x.strip(), os.getenv('LIBS', '').split('-l')[1:])))
+        library_dirs = list(map(lambda x: x.strip(), os.getenv('MODEL_LDFLAGS', '').split('-L')[1:])),
+        libraries = list(map(lambda x: x.strip(), os.getenv('MODEL_LIBS', '').split('-l')[1:])))

 utils = Extension('deepspeech._utils',
        ['python/utils.i'],
        include_dirs = [numpy_include],
+        library_dirs = list(map(lambda x: x.strip(), os.getenv('UTILS_LDFLAGS', '').split('-L')[1:])),
        libraries = ['deepspeech_utils'])

 setup(name = project_name,
--- a/taskcluster/.build.yml
+++ b/taskcluster/.build.yml
@ -30,3 +30,4 @@ build:
  args:
    tests_cmdline: ''
  python_wheel: ''
+  deepspeech_pkg_name: ''
--- a/taskcluster/test-linux-opt-base.tyml
+++ b/taskcluster/test-linux-opt-base.tyml
@ -39,6 +39,7 @@ then:
      in:
        TENSORFLOW_WHEEL: ${build.python_wheel}
        DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public
+        DEEPSPEECH_PYTHON_PACKAGE: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public/${build.deepspeech_pkg_name}
        DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
        DEEPSPEECH_AOT_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_aot_test}/artifacts/public
        DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz
--- a/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml
+++ b/taskcluster/test-training+deepspeech-linux-amd64-py27-opt.yml
@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-ctc-opt"
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 mozilla deepspeech"
+  python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/tensorflow_warpctc-1.4.0-cp27-cp27mu-linux_x86_64.whl'
+  deepspeech_pkg_name: 'deepspeech-0.1.0-cp27-cp27mu-manylinux1_x86_64.whl'
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU training Py2.7 (DS)"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 2.7 with DeepSpeech package installed, CPU only, optimized version"
--- a/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml
+++ b/taskcluster/test-training+deepspeech-linux-amd64-py36-opt.yml
@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-ctc-opt"
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 mozilla deepspeech"
+  python_wheel: 'https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu-py36/artifacts/public/tensorflow_warpctc-1.4.0-cp36-cp36m-linux_x86_64.whl'
+  deepspeech_pkg_name: 'deepspeech-0.1.0-cp36-cp36m-manylinux1_x86_64.whl'
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU training Py3.6 (DS)"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6 with DeepSpeech package installed, CPU only, optimized version"
--- a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml
+++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py27-opt.yml
@ -0,0 +1,14 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-ctc-opt"
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 upstream deepspeech"
+  deepspeech_pkg_name: 'deepspeech-0.1.0-cp27-cp27mu-manylinux1_x86_64.whl'
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 (DS)"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 with DeepSpeech package installed, CPU only, optimized version"
--- a/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml
+++ b/taskcluster/test-training_upstream+deepspeech-linux-amd64-py36-opt.yml
@ -0,0 +1,14 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-ctc-opt"
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.2 upstream deepspeech"
+  deepspeech_pkg_name: 'deepspeech-0.1.0-cp36-cp36m-manylinux1_x86_64.whl'
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU upstream training Py3.6 (DS)"
+    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 3.6 with DeepSpeech package installed, CPU only, optimized version"
--- a/tc-train-tests.sh
+++ b/tc-train-tests.sh
@ -6,6 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh

 pyver=$1
 tf=$2
+ds=$3

 if [ -z "${pyver}" ]; then
    echo "No python version given, aborting."
@ -38,6 +39,11 @@ if [ "${tf}" = "upstream" ]; then
    pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt
 fi;

+if [ "${ds}" = "deepspeech" ]; then
+    pip install "${DEEPSPEECH_PYTHON_PACKAGE}"
+    python -c "import tensorflow; from deepspeech.utils import audioToInputVector"
+fi;
+
 download_ctc_kenlm "/tmp/ds"

 pushd ${HOME}/DeepSpeech/ds/