#!/bin/bash set -xe install_pyenv() { if [ -z "${PYENV_ROOT}" ]; then echo "No PYENV_ROOT set"; exit 1; fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then mkdir -p "${PYENV_ROOT}/versions/" return; fi # Allows updating local cache if required if [ ! -e "${PYENV_ROOT}/bin/pyenv" ]; then git clone --quiet https://github.com/pyenv/pyenv.git ${PYENV_ROOT} else pushd ${PYENV_ROOT} git fetch origin popd fi pushd ${PYENV_ROOT} git checkout --quiet 20a1f0cd7a3d2f95800d8e0d5863b4e98f25f4df popd if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then git clone https://github.com/s1341/pyenv-alias.git ${PYENV_ROOT}/plugins/pyenv-alias pushd ${PYENV_ROOT}/plugins/pyenv-alias git checkout --quiet 8896eebb5b47389249b35d21d8a5e74aa33aff08 popd fi eval "$(pyenv init -)" } install_pyenv_virtualenv() { local PYENV_VENV=$1 if [ -z "${PYENV_VENV}" ]; then echo "No PYENV_VENV set"; exit 1; fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then echo "No pyenv virtualenv support ; will install virtualenv locally from pip" return fi; if [ ! -e "${PYENV_VENV}/bin/pyenv-virtualenv" ]; then git clone --quiet https://github.com/pyenv/pyenv-virtualenv.git ${PYENV_VENV} pushd ${PYENV_VENV} git checkout --quiet 5419dc732066b035a28680475acd7b661c7c397d popd fi; eval "$(pyenv virtualenv-init -)" } setup_pyenv_virtualenv() { local version=$1 local name=$2 if [ -z "${PYENV_ROOT}" ]; then echo "No PYENV_ROOT set"; exit 1; fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then echo "should setup virtualenv ${name} for ${version}" mkdir ${PYENV_ROOT}/versions/${version}/envs PATH=${PYENV_ROOT}/versions/${version}/tools:${PYENV_ROOT}/versions/${version}/tools/Scripts:$PATH python -m venv ${PYENV_ROOT}/versions/${version}/envs/${name} else ls -hal "${PYENV_ROOT}/versions/" # There could be a symlink when re-using cacche on macOS # We don't care, let's just remove it if [ -L "${PYENV_ROOT}/versions/${name}" ]; then rm "${PYENV_ROOT}/versions/${name}" fi # Don't force-reinstall existing version if [ ! -f "${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate" ]; then pyenv virtualenv ${version} ${name} fi fi } virtualenv_activate() { local version=$1 local name=$2 if [ -z "${PYENV_ROOT}" ]; then echo "No PYENV_ROOT set"; exit 1; fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then source ${PYENV_ROOT}/versions/${version}/envs/${name}/Scripts/activate else source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate fi } virtualenv_deactivate() { local version=$1 local name=$2 if [ -z "${PYENV_ROOT}" ]; then echo "No PYENV_ROOT set"; exit 1; fi; deactivate if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then rm -fr ${PYENV_ROOT}/versions/${version}/ else pyenv uninstall --force ${name} fi } pyenv_install() { local version=$1 local version_alias=$2 if [ -z "${version_alias}" ]; then echo "WARNING, no version_alias specified, please ensure call site is okay" version_alias=${version} fi; if [ -z "${PYENV_ROOT}" ]; then echo "No PYENV_ROOT set"; exit 1; fi; if [ -z "${DS_CPU_COUNT}" ]; then echo "No idea of parallelism"; exit 1; fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then PATH=$(cygpath ${ChocolateyInstall})/bin:$PATH nuget install python -Version ${version} -OutputDirectory ${PYENV_ROOT}/versions/ mv ${PYENV_ROOT}/versions/python.${version} ${PYENV_ROOT}/versions/${version_alias} PY_TOOLS_DIR="$(cygpath -w ${PYENV_ROOT}/versions/${version_alias}/tools/)" TEMP=$(cygpath -w ${DS_ROOT_TASK}/tmp/) PATH=${PY_TOOLS_DIR}:$PATH python -m pip uninstall pip -y PATH=${PY_TOOLS_DIR}:$PATH python -m ensurepip pushd ${PYENV_ROOT}/versions/${version_alias}/tools/Scripts/ ln -s pip3.exe pip.exe popd else # If there's already a matching directory, we should re-use it # otherwise, pyenv install will force-rebuild ls -hal "${PYENV_ROOT}/versions/${version_alias}/" || true if [ ! -d "${PYENV_ROOT}/versions/${version_alias}/" ]; then VERSION_ALIAS=${version_alias} MAKEOPTS=-j${DS_CPU_COUNT} pyenv install ${version} fi; fi } # Hack to extract Ubuntu's 16.04 libssl 1.0.2 packages and use them during the # local build of Python. # # Avoid (risky) upgrade of base system, allowing to keep one task build that # builds all the python packages maybe_ssl102_py37() { pyver=$1 unset PY37_OPENSSL unset PY37_LDPATH unset PY37_SOURCE_PACKAGE ARCH=$(uname -m) case "${pyver}" in 3.7*|3.8*) if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial if [ -d "${PY37_OPENSSL_DIR}" ]; then rm -rf "${PY37_OPENSSL_DIR}" fi mkdir -p ${PY37_OPENSSL_DIR} ${WGET} -P ${TASKCLUSTER_TMP_DIR} \ http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.0.2g-1ubuntu4.15_amd64.deb \ http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.0.0_1.0.2g-1ubuntu4.15_amd64.deb for deb in ${TASKCLUSTER_TMP_DIR}/libssl*.deb; do dpkg -x ${deb} ${PY37_OPENSSL_DIR} done; # Python configure expects things to be under lib/ mv ${PY37_OPENSSL_DIR}/usr/include/x86_64-linux-gnu/openssl/opensslconf.h ${PY37_OPENSSL_DIR}/usr/include/openssl/ mv ${PY37_OPENSSL_DIR}/lib/x86_64-linux-gnu/lib* ${PY37_OPENSSL_DIR}/usr/lib/ mv ${PY37_OPENSSL_DIR}/usr/lib/x86_64-linux-gnu/* ${PY37_OPENSSL_DIR}/usr/lib/ ln -sfn libcrypto.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libcrypto.so ln -sfn libssl.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libssl.so export PY37_OPENSSL="--with-openssl=${PY37_OPENSSL_DIR}/usr" export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" fi; case "${pyver}" in 3.7*) export NUMPY_BUILD_VERSION="==1.14.5" export NUMPY_DEP_VERSION=">=1.14.5" ;; 3.8*) export NUMPY_BUILD_VERSION="==1.17.3" export NUMPY_DEP_VERSION=">=1.17.3" ;; esac ;; esac } maybe_numpy_min_version_winamd64() { local pyver=$1 if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then return; fi # We set >= and < to make sure we have no numpy incompatibilities # otherwise, `from deepspeech.impl` throws with "illegal instruction" case "${pyver}" in 3.5*) export NUMPY_BUILD_VERSION="==1.11.0" export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0" ;; 3.6*) export NUMPY_BUILD_VERSION="==1.12.0" export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5" ;; 3.7*) export NUMPY_BUILD_VERSION="==1.14.5" export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" ;; 3.8*) export NUMPY_BUILD_VERSION="==1.17.3" export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" ;; esac } get_python_pkg_url() { local pyver_pkg=$1 local py_unicode_type=$2 local pkgname=$3 if [ -z "${pkgname}" ]; then pkgname="deepspeech" fi local root=$4 if [ -z "${root}" ]; then root="${DEEPSPEECH_ARTIFACTS_ROOT}" fi local platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; plat = "win" if plat == "windows" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine().lower()));') local whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" local deepspeech_pkg="${pkgname}-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" echo "${root}/${deepspeech_pkg}" } extract_python_versions() { # call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf pyalias local _pyver_full=$1 if [ -z "${_pyver_full}" ]; then echo "No python version given, aborting." exit 1 fi; local _pyver=$(echo "${_pyver_full}" | cut -d':' -f1) # 3.8.x => 38 local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.') # UCS2 => narrow unicode # UCS4 => wide unicode local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2) if [ "${_py_unicode_type}" = "m" ]; then local _pyconf="ucs2" elif [ "${_py_unicode_type}" = "mu" ]; then local _pyconf="ucs4" elif [ "${_py_unicode_type}" = "" ]; then # valid for Python 3.8 local _pyconf="ucs4" fi; local _pyalias="${_pyver}_${_pyconf}" eval "${2}=${_pyver}" eval "${3}=${_pyver_pkg}" eval "${4}=${_py_unicode_type}" eval "${5}=${_pyconf}" eval "${6}=${_pyalias}" }