mirror of
https://github.com/mozilla/DeepSpeech.git
synced 2025-10-26 11:19:39 +00:00
125 lines
3.6 KiB
Bash
Executable File
125 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -xe
|
|
|
|
generic_download_tarxz()
|
|
{
|
|
target_dir=$1
|
|
url=$2
|
|
|
|
if [ -z "${target_dir}" -o -z "${url}" ]; then
|
|
echo "Empty name for target directory or URL:"
|
|
echo " target_dir=${target_dir}"
|
|
echo " url=${url}"
|
|
exit 1
|
|
fi;
|
|
|
|
mkdir -p ${target_dir} || true
|
|
|
|
${WGET} ${url} -O - | ${UNXZ} | ${TAR} -C ${target_dir} -xf -
|
|
}
|
|
|
|
download_native_client_files()
|
|
{
|
|
generic_download_tarxz "$1" "${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz"
|
|
}
|
|
|
|
set_ldc_sample_filename()
|
|
{
|
|
local _bitrate=$1
|
|
|
|
if [ -z "${_bitrate}" ]; then
|
|
echo "Bitrate should not be empty"
|
|
exit 1
|
|
fi;
|
|
|
|
case "${_bitrate}" in
|
|
8k)
|
|
ldc93s1_sample_filename='LDC93S1_pcms16le_1_8000.wav'
|
|
;;
|
|
16k)
|
|
ldc93s1_sample_filename='LDC93S1_pcms16le_1_16000.wav'
|
|
;;
|
|
esac
|
|
}
|
|
|
|
download_data()
|
|
{
|
|
${WGET} -P "${TASKCLUSTER_TMP_DIR}" "${model_source}"
|
|
${WGET} -P "${TASKCLUSTER_TMP_DIR}" "${model_source_mmap}"
|
|
cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/smoke_test/*.wav ${TASKCLUSTER_TMP_DIR}/
|
|
cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/smoke_test/pruned_lm.scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer
|
|
cp -R ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/test ${TASKCLUSTER_TMP_DIR}/test_sources
|
|
}
|
|
|
|
download_material()
|
|
{
|
|
target_dir=$1
|
|
|
|
download_native_client_files "${target_dir}"
|
|
download_data
|
|
|
|
ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav
|
|
}
|
|
|
|
maybe_install_xldd()
|
|
{
|
|
# -s required to avoid the noisy output like "Entering / Leaving directories"
|
|
toolchain=$(make -s -C ${DS_DSDIR}/native_client/ TARGET=${SYSTEM_TARGET} TFDIR=${DS_TFDIR} print-toolchain)
|
|
if [ ! -x "${toolchain}ldd" ]; then
|
|
cp "${DS_DSDIR}/native_client/xldd" "${toolchain}ldd" && chmod +x "${toolchain}ldd"
|
|
fi
|
|
}
|
|
|
|
# Checks whether we run a patched version of bazel.
|
|
# Patching is required to dump computeKey() parameters to .ckd files
|
|
# See bazel.patch
|
|
# Return 0 (success exit code) on patched version, 1 on release version
|
|
is_patched_bazel()
|
|
{
|
|
bazel_version=$(bazel version | grep 'Build label:' | cut -d':' -f2)
|
|
|
|
bazel shutdown
|
|
|
|
if [ -z "${bazel_version}" ]; then
|
|
return 0;
|
|
else
|
|
return 1;
|
|
fi;
|
|
}
|
|
|
|
verify_bazel_rebuild()
|
|
{
|
|
bazel_explain_file="$1"
|
|
|
|
if [ ! -f "${bazel_explain_file}" ]; then
|
|
echo "No such explain file: ${bazel_explain_file}"
|
|
exit 1
|
|
fi;
|
|
|
|
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
|
|
|
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
|
|
|
spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libdeepspeech.so' | wc -l)
|
|
if [ "${spurious_rebuilds}" -ne 0 ]; then
|
|
echo "Bazel rebuilds some file it should not, please check."
|
|
|
|
if is_patched_bazel; then
|
|
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/ckd/ds ${DS_ROOT_TASK}/DeepSpeech/ckd/tf
|
|
tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
|
|
tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/
|
|
|
|
echo "Making a diff between CKD files"
|
|
mkdir -p ${TASKCLUSTER_ARTIFACTS}
|
|
diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
|
|
|
|
rm -fr ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
|
|
else
|
|
echo "Cannot get CKD information from release, please use patched Bazel"
|
|
fi;
|
|
|
|
exit 1
|
|
fi;
|
|
}
|