From 06d4379994452e7eb72c839508f185b2ebc32b86 Mon Sep 17 00:00:00 2001 From: daanzu Date: Mon, 1 Oct 2018 10:02:07 -0400 Subject: [PATCH 01/18] Fix docstrings that were being missed by Sphinx --- DeepSpeech.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/DeepSpeech.py b/DeepSpeech.py index a12b190d..04383ef7 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -875,15 +875,15 @@ def new_id(): return id_counter class Sample(object): - def __init__(self, src, res, loss, mean_edit_distance, sample_wer): - '''Represents one item of a WER report. + '''Represents one item of a WER report. - Args: - src (str): source text - res (str): resulting text - loss (float): computed loss of this item - mean_edit_distance (float): computed mean edit distance of this item - ''' + Args: + src (str): source text + res (str): resulting text + loss (float): computed loss of this item + mean_edit_distance (float): computed mean edit distance of this item + ''' + def __init__(self, src, res, loss, mean_edit_distance, sample_wer): self.src = src self.res = res self.loss = loss @@ -894,16 +894,16 @@ class Sample(object): return 'WER: %f, loss: %f, mean edit distance: %f\n - src: "%s"\n - res: "%s"' % (self.wer, self.loss, self.mean_edit_distance, self.src, self.res) class WorkerJob(object): - def __init__(self, epoch_id, index, set_name, steps, report): - '''Represents a job that should be executed by a worker. + '''Represents a job that should be executed by a worker. - Args: - epoch_id (int): the ID of the 'parent' epoch - index (int): the epoch index of the 'parent' epoch - set_name (str): the name of the data-set - one of 'train', 'dev', 'test' - steps (int): the number of `session.run` calls - report (bool): if this job should produce a WER report - ''' + Args: + epoch_id (int): the ID of the 'parent' epoch + index (int): the epoch index of the 'parent' epoch + set_name (str): the name of the data-set - one of 'train', 'dev', 'test' + steps (int): the number of `session.run` calls + report (bool): if this job should produce a WER report + ''' + def __init__(self, epoch_id, index, set_name, steps, report): self.id = new_id() self.epoch_id = epoch_id self.index = index @@ -1075,6 +1075,12 @@ class Epoch(object): class TrainingCoordinator(object): + ''' Central training coordination class. + Used for distributing jobs among workers of a cluster. + Instantiated on all workers, calls of non-chief workers will transparently + HTTP-forwarded to the chief worker instance. + ''' + class TrainingCoordinationHandler(BaseHTTPServer.BaseHTTPRequestHandler): '''Handles HTTP requests from remote workers to the Training Coordinator. ''' @@ -1119,13 +1125,7 @@ class TrainingCoordinator(object): ''' return - def __init__(self): - ''' Central training coordination class. - Used for distributing jobs among workers of a cluster. - Instantiated on all workers, calls of non-chief workers will transparently - HTTP-forwarded to the chief worker instance. - ''' self._init() self._lock = Lock() self.started = False From e00c63c0a8028d32bae30e68bc461478f486e535 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 11 Oct 2018 15:58:37 -0300 Subject: [PATCH 02/18] Fix evaluate.py to handle compact format in cached features --- evaluate.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/evaluate.py b/evaluate.py index d4842d62..c1a2d45a 100755 --- a/evaluate.py +++ b/evaluate.py @@ -99,12 +99,31 @@ def main(_): # sort examples by length, improves packing of batches and timesteps test_data = preprocess( - FLAGS.test_files, + FLAGS.test_files.split(','), FLAGS.test_batch_size, - hdf5_dest_path=FLAGS.hdf5_test_set).sort_values( + alphabet=alphabet, + numcep=N_FEATURES, + numcontext=N_CONTEXT, + hdf5_cache_path=FLAGS.hdf5_test_set).sort_values( by="features_len", ascending=False) + def create_windows(features): + num_strides = len(features) - (N_CONTEXT * 2) + + # Create a view into the array with overlapping strides of size + # numcontext (past) + 1 (present) + numcontext (future) + window_size = 2*N_CONTEXT+1 + features = np.lib.stride_tricks.as_strided( + features, + (num_strides, window_size, N_FEATURES), + (features.strides[0], features.strides[0], features.strides[1]), + writeable=False) + + return features + + test_data['features'] = test_data['features'].apply(create_windows) + with tf.Session() as session: inputs, outputs = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=N_STEPS) @@ -158,7 +177,7 @@ def main(_): logits = np.empty([0, FLAGS.test_batch_size, alphabet.size() + 1]) for i in range(0, batch_features.shape[1], N_STEPS): - chunk_features = batch_features[:, i:i + N_STEPS, :] + chunk_features = batch_features[:, i:i + N_STEPS, :, :] chunk_features_len = np.minimum( batch_features_len, full_step_len) @@ -168,6 +187,7 @@ def main(_): chunk_features = np.pad(chunk_features, ((0, 0), (0, FLAGS.n_steps - steps_in_chunk), + (0, 0), (0, 0)), mode='constant', constant_values=0) From ed489b1112d093d2aed2b696c522d85cc64a46f2 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Fri, 12 Oct 2018 13:59:54 +0200 Subject: [PATCH 03/18] Remove useless bogus cURL call --- taskcluster/docker-build-base.tyml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/taskcluster/docker-build-base.tyml b/taskcluster/docker-build-base.tyml index 48ec93ee..d9de927d 100644 --- a/taskcluster/docker-build-base.tyml +++ b/taskcluster/docker-build-base.tyml @@ -43,8 +43,7 @@ then: add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && apt-get -qq update && apt-get -qq -y install docker-ce && mkdir -p /opt/deepspeech && git clone --quiet ${event.head.repo.url} /opt/deepspeech && cd /opt/deepspeech && git checkout --quiet ${event.head.sha} && - curl -fsSL "https://raw.githubusercontent.com/${event.head.user.login}/${event.head.repo.name}/${event.head.sha}/${dockerfile}" > ${dockerfile} && - docker build . + docker build --file ${dockerfile} . artifacts: "public": From c38dc099d36fd593c0ff27e475014bf72a32c94e Mon Sep 17 00:00:00 2001 From: kdavis-mozilla Date: Thu, 11 Oct 2018 12:23:24 +0200 Subject: [PATCH 04/18] Fixed #1638 (Update Hyperparameters for 0.3.0) --- DeepSpeech.py | 2 +- bin/run-tc-ldc93s1_new.sh | 2 +- examples/vad_transcriber/wavTranscriber.py | 4 +-- native_client/client.cc | 2 +- native_client/javascript/client.js | 2 +- native_client/python/client.py | 2 +- tc-tests-utils.sh | 30 ++++++++++++---------- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/DeepSpeech.py b/DeepSpeech.py index dd4ef113..c94868a5 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -162,7 +162,7 @@ def create_flags(): tf.app.flags.DEFINE_string ('lm_trie_path', 'data/lm/trie', 'path to the language model trie file created with native_client/generate_trie') tf.app.flags.DEFINE_integer ('beam_width', 1024, 'beam width used in the CTC decoder when building candidate transcriptions') tf.app.flags.DEFINE_float ('lm_weight', 1.50, 'the alpha hyperparameter of the CTC decoder. Language Model weight.') - tf.app.flags.DEFINE_float ('valid_word_count_weight', 2.25, 'valid word insertion weight. This is used to lessen the word insertion penalty when the inserted word is part of the vocabulary.') + tf.app.flags.DEFINE_float ('valid_word_count_weight', 2.10, 'valid word insertion weight. This is used to lessen the word insertion penalty when the inserted word is part of the vocabulary.') # Inference mode diff --git a/bin/run-tc-ldc93s1_new.sh b/bin/run-tc-ldc93s1_new.sh index 524c9f06..1aabd98d 100755 --- a/bin/run-tc-ldc93s1_new.sh +++ b/bin/run-tc-ldc93s1_new.sh @@ -14,7 +14,7 @@ python -u DeepSpeech.py \ --train_files ${ldc93s1_csv} --train_batch_size 1 \ --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch 85 --random_seed 4567 --default_stddev 0.046875 \ + --n_hidden 494 --epoch 105 --random_seed 4567 --default_stddev 0.046875 \ --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \ --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/train' \ --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \ diff --git a/examples/vad_transcriber/wavTranscriber.py b/examples/vad_transcriber/wavTranscriber.py index 5fdcb394..b3f0d272 100644 --- a/examples/vad_transcriber/wavTranscriber.py +++ b/examples/vad_transcriber/wavTranscriber.py @@ -19,8 +19,8 @@ def load_model(models, alphabet, lm, trie): N_FEATURES = 26 N_CONTEXT = 9 BEAM_WIDTH = 500 - LM_WEIGHT = 1.75 - VALID_WORD_COUNT_WEIGHT = 1.00 + LM_WEIGHT = 1.50 + VALID_WORD_COUNT_WEIGHT = 2.10 model_load_start = timer() ds = Model(models, N_FEATURES, N_CONTEXT, alphabet, BEAM_WIDTH) diff --git a/native_client/client.cc b/native_client/client.cc index b2cda25d..f6220140 100644 --- a/native_client/client.cc +++ b/native_client/client.cc @@ -23,7 +23,7 @@ #define N_CONTEXT 9 #define BEAM_WIDTH 500 #define LM_WEIGHT 1.50f -#define VALID_WORD_COUNT_WEIGHT 2.25f +#define VALID_WORD_COUNT_WEIGHT 2.10f typedef struct { const char* string; diff --git a/native_client/javascript/client.js b/native_client/javascript/client.js index 3f6c78ac..1e92379f 100644 --- a/native_client/javascript/client.js +++ b/native_client/javascript/client.js @@ -19,7 +19,7 @@ const LM_WEIGHT = 1.50; // Valid word insertion weight. This is used to lessen the word insertion penalty // when the inserted word is part of the vocabulary -const VALID_WORD_COUNT_WEIGHT = 2.25; +const VALID_WORD_COUNT_WEIGHT = 2.10; // These constants are tied to the shape of the graph used (changing them changes diff --git a/native_client/python/client.py b/native_client/python/client.py index 7becb3e0..60e8097c 100644 --- a/native_client/python/client.py +++ b/native_client/python/client.py @@ -27,7 +27,7 @@ LM_WEIGHT = 1.50 # Valid word insertion weight. This is used to lessen the word insertion penalty # when the inserted word is part of the vocabulary -VALID_WORD_COUNT_WEIGHT = 2.25 +VALID_WORD_COUNT_WEIGHT = 2.10 # These constants are tied to the shape of the graph used (changing them changes diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 93a8fa0c..525e3683 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -135,26 +135,30 @@ assert_shows_something() assert_correct_ldc93s1() { - # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS - # assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" - assert_working_inference "$1" "she had" + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" +} + +assert_correct_ldc93s1_lm() +{ + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" } assert_correct_multi_ldc93s1() { - # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS - #assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%" - #assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%" - return 0 + assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%" + assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%" ## 8k will output garbage anyway ... # assert_shows_something "$1" "/LDC93S1_pcms16le_1_8000.wav%she hayorasryrtl lyreasy asr watal w water all year%" } assert_correct_ldc93s1_prodmodel() { - # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS - #assert_correct_inference "$1" "she had tired or so and greasy wash war or year" - assert_working_inference "$1" "she had" + assert_correct_inference "$1" "she hadered or so and greasy wash war or year" +} + +assert_correct_ldc93s1_prodmodel_stereo_44k() +{ + assert_correct_inference "$1" "she had tered or so and greasy wash war or year" } assert_correct_warning_upsampling() @@ -185,13 +189,13 @@ run_all_inference_tests() assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_withlm}" + assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_withlm_stereo_44k}" + assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}" phrase_pbmodel_nolm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}" @@ -209,7 +213,7 @@ run_prod_inference_tests() assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) - assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm_stereo_44k}" + assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" From 0fbf61cd6c7e0cbdb1c49b3794298e5477794b92 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Sat, 13 Oct 2018 11:08:32 +0200 Subject: [PATCH 05/18] Bump to v0.3.0-alpha.1 X-DeepSpeech: NOBUILD --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 07bf7a73..1a760289 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0-alpha.0 +0.3.0-alpha.1 From 609120f3da80d9622fc14803b05faf4bf476013a Mon Sep 17 00:00:00 2001 From: Nicolas Panel Date: Sun, 14 Oct 2018 13:48:59 +0200 Subject: [PATCH 06/18] [TrainingSpeech importer] stick to data/alphabet.txt aims to generate english-complatible dataset since no FR alphabet.txt for now see https://github.com/mozilla/DeepSpeech/pull/1599#issuecomment-426544379 for more info --- bin/import_ts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/import_ts.py b/bin/import_ts.py index a2d061ca..51d91153 100755 --- a/bin/import_ts.py +++ b/bin/import_ts.py @@ -11,6 +11,7 @@ import sys sys.path.insert(1, os.path.join(sys.path[0], '..')) import csv +import unidecode import zipfile from os import path @@ -96,7 +97,7 @@ def cleanup_transcript(text): text = text.replace('’', "'").replace('\u00A0', ' ') text = PUNCTUATIONS_REG.sub(' ', text) text = MULTIPLE_SPACES_REG.sub(' ', text) - return text.strip().lower() + return unidecode.unidecode(text).strip().lower() if __name__ == "__main__": From 61d9193ce52503b52954736d1e60761c32d0b339 Mon Sep 17 00:00:00 2001 From: nicolaspanel Date: Mon, 15 Oct 2018 10:18:14 +0200 Subject: [PATCH 07/18] [TrainingSpeech importer] add `--english-compatible` option --- bin/import_ts.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/bin/import_ts.py b/bin/import_ts.py index 51d91153..a6f1ff00 100755 --- a/bin/import_ts.py +++ b/bin/import_ts.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, division, print_function # Make sure we can import stuff from util/ # This script needs to be run from the root of the DeepSpeech repository +import argparse import os import re import sys @@ -26,7 +27,7 @@ ARCHIVE_DIR_NAME = 'ts_' + ARCHIVE_NAME ARCHIVE_URL = 'https://s3.eu-west-3.amazonaws.com/audiocorp/releases/' + ARCHIVE_NAME + '.zip' -def _download_and_preprocess_data(target_dir): +def _download_and_preprocess_data(target_dir, english_compatible=False): # Making path absolute target_dir = path.abspath(target_dir) # Conditionally download data @@ -34,7 +35,8 @@ def _download_and_preprocess_data(target_dir): # Conditionally extract archive data _maybe_extract(target_dir, ARCHIVE_DIR_NAME, archive_path) # Conditionally convert TrainingSpeech data to DeepSpeech CSVs and wav - _maybe_convert_sets(target_dir, ARCHIVE_DIR_NAME) + _maybe_convert_sets(target_dir, ARCHIVE_DIR_NAME, english_compatible=english_compatible) + def _maybe_extract(target_dir, extracted_data, archive_path): # If target_dir/extracted_data does not exist, extract archive in target_dir @@ -48,7 +50,8 @@ def _maybe_extract(target_dir, extracted_data, archive_path): else: print('Found directory "%s" - not extracting it from archive.' % archive_path) -def _maybe_convert_sets(target_dir, extracted_data): + +def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False): extracted_dir = path.join(target_dir, extracted_data) # override existing CSV with normalized one target_csv_template = os.path.join(target_dir, 'ts_' + ARCHIVE_NAME + '_{}.csv') @@ -71,7 +74,7 @@ def _maybe_convert_sets(target_dir, extracted_data): test_writer.writeheader() for i, item in enumerate(data): - transcript = validate_label(cleanup_transcript(item['text'])) + transcript = validate_label(cleanup_transcript(item['text'], english_compatible=english_compatible)) if not transcript: continue wav_filename = os.path.join(target_dir, extracted_data, item['path']) @@ -93,12 +96,22 @@ PUNCTUATIONS_REG = re.compile(r"[°\-,;!?.()\[\]*…—]") MULTIPLE_SPACES_REG = re.compile(r'\s{2,}') -def cleanup_transcript(text): +def cleanup_transcript(text, english_compatible=False): text = text.replace('’', "'").replace('\u00A0', ' ') text = PUNCTUATIONS_REG.sub(' ', text) text = MULTIPLE_SPACES_REG.sub(' ', text) - return unidecode.unidecode(text).strip().lower() + if english_compatible: + text = unidecode.unidecode(text) + return text.strip().lower() + + +def handle_args(): + parser = argparse.ArgumentParser(description='Importer for TrainingSpeech dataset.') + parser.add_argument(dest='target_dir') + parser.add_argument('--english-compatible', action='store_true', dest='english_compatible', help='Remove diactrics and other non-ascii chars.') + return parser.parse_args() if __name__ == "__main__": - _download_and_preprocess_data(sys.argv[1]) + cli_args = handle_args() + _download_and_preprocess_data(cli_args.target_dir, cli_args.english_compatible) From 111262a3ec3c1fe1ebe0b8d414cede81e6f23864 Mon Sep 17 00:00:00 2001 From: kdavis-mozilla Date: Mon, 15 Oct 2018 16:52:21 +0200 Subject: [PATCH 08/18] Fixed #1649 (Update README's for 0.3.0) --- README.md | 2 +- examples/vad_transcriber/wavTranscription.md | 4 ++-- native_client/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3c1ed668..8ba68947 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ git clone https://github.com/mozilla/DeepSpeech If you want to use the pre-trained English model for performing speech-to-text, you can download it (along with other important inference material) from the [DeepSpeech releases page](https://github.com/mozilla/DeepSpeech/releases). Alternatively, you can run the following command to download and unzip the files in your current directory: ```bash -wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.2.0/deepspeech-0.2.0-models.tar.gz | tar xvfz - +wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.3.0/deepspeech-0.3.0-models.tar.gz | tar xvfz - ``` ## Using the model diff --git a/examples/vad_transcriber/wavTranscription.md b/examples/vad_transcriber/wavTranscription.md index e34c1c75..efa7839b 100644 --- a/examples/vad_transcriber/wavTranscription.md +++ b/examples/vad_transcriber/wavTranscription.md @@ -26,7 +26,7 @@ Set the aggressiveness mode, to an integer between 0 and 3. ``` (venv) ~/Deepspeech/examples/vad_transcriber -$ python3 audioTranscript_cmd.py --aggressive 1 --audio ./audio/guido-van-rossum.wav --model ./models/0.2.0/ +$ python3 audioTranscript_cmd.py --aggressive 1 --audio ./audio/guido-van-rossum.wav --model ./models/0.3.0/ Filename Duration(s) Inference Time(s) Model Load Time(s) LM Load Time(s) @@ -58,4 +58,4 @@ In such a scenario, the GUI tool will not work. The following steps is known to (venv) ~/Deepspeech/examples/vad_transcriber$ export PYTHONPATH=/usr/lib/python3/dist-packages/ (venv) ~/Deepspeech/examples/vad_transcriber$ python3 audioTranscript_gui.py -``` \ No newline at end of file +``` diff --git a/native_client/README.md b/native_client/README.md index 51e65676..ba43d328 100644 --- a/native_client/README.md +++ b/native_client/README.md @@ -133,4 +133,4 @@ make package make npm-pack ``` -This will create the package `deepspeech-0.2.0.tgz` in `native_client/javascript`. +This will create the package `deepspeech-0.3.0.tgz` in `native_client/javascript`. From 32a75d1d9d81abc3e23d51a7a0cd658e324d21c4 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Mon, 15 Oct 2018 14:24:20 +0200 Subject: [PATCH 09/18] Avoid task_TIMESTAMP use on macOS generic-worker --- taskcluster/darwin-opt-base.tyml | 53 +++++---------------------- taskcluster/test-darwin-opt-base.tyml | 17 ++++++++- tc-brew-tests.sh | 2 +- 3 files changed, 26 insertions(+), 46 deletions(-) diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index fe1b4e36..cd398974 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -79,54 +79,16 @@ payload: - "-cxe" - > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && + (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) && + (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) && export TASKCLUSTER_TASK_DIR="$(pwd)" && export LC_ALL=C && - export TASKCLUSTER_ORIGIN_PATH="${system.homedir.osx}/TaskCluster/.*/task_[[:digit:]]\{10\}" && - export TASKCLUSTER_REWRITE_PATH="$TASKCLUSTER_TASK_DIR" && export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" && export MACOSX_DEPLOYMENT_TARGET=10.10 && env && - ( - ( - for link in $(find -L "$TASKCLUSTER_TASK_DIR/homebrew/" -type l); - do - newloc=$(readlink "$link" | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g"); - ln -hfs "$newloc" "$link"; - done) || true) && - ( - ( - for lib in $(find "$TASKCLUSTER_TASK_DIR/homebrew/" -type f -name "*.dylib"); - do - chmod +w "$lib"; - for id in $(otool -D "$lib" | grep "$TASKCLUSTER_ORIGIN_PATH"); - do - newid=$(echo $id | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g"); - install_name_tool -id "$newid" "$lib"; - done; - - for dep in $(otool -L "$lib" | awk '{ print $1 }' | grep "$TASKCLUSTER_ORIGIN_PATH"); - do - newdep=$(echo $dep | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g"); - install_name_tool -change "$dep" "$newdep" "$lib"; - done; - chmod -w "$lib"; - otool -L "$lib"; - done) || true) && - ( - ( - for bin in $(find "$TASKCLUSTER_TASK_DIR/homebrew/" -perm +111 -type f); - do - chmod +w "$bin"; - for dep in $(otool -L "$bin" | awk '{ print $1 }' | grep "$TASKCLUSTER_ORIGIN_PATH"); - do - newdep=$(echo $dep | sed -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g"); - install_name_tool -change "$dep" "$newdep" "$bin"; - done; - chmod -w "$bin"; - otool -L "$bin"; - done) || true) && - (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR --transform "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g" -xf - ) && - ((grep -R -I --files-with-matches "$TASKCLUSTER_ORIGIN_PATH" $TASKCLUSTER_TASK_DIR | grep -v "$TASKCLUSTER_TASK_DIR/generic-worker/" | xargs -n 1 -P 16 sed -i '' -e "s|$TASKCLUSTER_ORIGIN_PATH|$TASKCLUSTER_REWRITE_PATH|g") || true) && + (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && @@ -134,7 +96,10 @@ payload: $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/tc-brew.sh && ${swig.patch_nodejs.osx} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ; + mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && + mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ artifacts: - type: "directory" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index efe5d56d..d2d2a5ac 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -23,6 +23,8 @@ then: scopes: [ "queue:route:notify.irc-channel.*", + "generic-worker:cache:deepspeech-homebrew-bin", + "generic-worker:cache:deepspeech-homebrew-cache", ] payload: @@ -49,6 +51,10 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && + (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) && + (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) && export TASKCLUSTER_TASK_DIR="$(pwd)" && export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" && export LC_ALL=C && @@ -59,7 +65,16 @@ then: cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && cd $TASKCLUSTER_TASK_DIR && source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-brew-tests.sh && ${extraSystemSetup} && - /bin/bash ${build.args.tests_cmdline} + /bin/bash ${build.args.tests_cmdline} ; + mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && + mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ + + mounts: + - cacheName: deepspeech-homebrew-bin + directory: homebrew/ + - cacheName: deepspeech-homebrew-cache + directory: homebrew.cache/ metadata: name: ${build.metadata.name} diff --git a/tc-brew-tests.sh b/tc-brew-tests.sh index d1713ec9..fa51d5cb 100644 --- a/tc-brew-tests.sh +++ b/tc-brew-tests.sh @@ -37,7 +37,7 @@ install_local_homebrew() mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}" mkdir -p "${HOMEBREW_CACHE}" - curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}" + curl -L https://github.com/Homebrew/brew/tarball/1.7.7 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}" export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then From a6f9a60967b2586eedfbc9d962496084c7b65050 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 16 Oct 2018 09:32:55 +0200 Subject: [PATCH 10/18] Update TensorFlow r1.11 --- taskcluster/darwin-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-ctc-opt.yml | 4 ++-- taskcluster/linux-amd64-gpu-opt.yml | 4 ++-- taskcluster/linux-arm64-cpu-opt.yml | 4 ++-- taskcluster/linux-rpi3-cpu-opt.yml | 4 ++-- taskcluster/node-package.yml | 4 ++-- taskcluster/test-armbian-opt-base.tyml | 2 +- taskcluster/test-darwin-opt-base.tyml | 2 +- taskcluster/test-linux-opt-base.tyml | 2 +- taskcluster/test-raspbian-opt-base.tyml | 2 +- taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index c1fbe87d..99ed3ea7 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -6,8 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.osx/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 3178a835..794b0c4d 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -14,8 +14,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index e933b7c5..04a337f2 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,8 +4,8 @@ build: - "pull_request.synchronize" - "pull_request.reopened" template_file: linux-opt-base.tyml - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 4b33fa19..8b3e1f08 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -12,8 +12,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.gpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.gpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/summarize_graph" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 1587f9ae..9326f3ef 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.arm64/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm64/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index b735ca7b..86998800 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index 59713bc6..f21709ff 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -16,8 +16,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index b463dfbf..4feb0f3b 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" command: - "/bin/bash" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index d2d2a5ac..039f54bb 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -41,7 +41,7 @@ then: DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" command: - - "/bin/bash" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 4ff04890..79a162bf 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -44,7 +44,7 @@ then: DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" command: - "/bin/bash" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index b8add444..71374453 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-7-g689b0da" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" command: - "/bin/bash" diff --git a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml index dfd5cd3a..8b910c5e 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu" - convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.689b0daf0bac618551e7dd94bd2045b7eded1458.cpu/artifacts/public/convert_graphdef_memmapped_format" + convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/convert_graphdef_memmapped_format" metadata: name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version" From 529ebdfcb4507c7ced8f42b2d386b8a780a39551 Mon Sep 17 00:00:00 2001 From: Faissal Bensefia Date: Fri, 19 Oct 2018 23:23:07 +0100 Subject: [PATCH 11/18] Fix import_cv.py I have no idea how this ever worked --- bin/import_cv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/import_cv.py b/bin/import_cv.py index 37a87496..4988671b 100755 --- a/bin/import_cv.py +++ b/bin/import_cv.py @@ -10,6 +10,7 @@ sys.path.insert(1, os.path.join(sys.path[0], '..')) import csv import tarfile import subprocess +import progressbar from glob import glob from os import path From 46b327a3d9a5ab57ddbe74d81eefe989bf6c6014 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 23 Oct 2018 13:00:11 +0200 Subject: [PATCH 12/18] Bump VERSION to v0.3.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1a760289..0d91a54c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0-alpha.1 +0.3.0 From 46d1cece4f98d726d88579500734a95686cfe16b Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Mon, 22 Oct 2018 11:34:16 -0300 Subject: [PATCH 13/18] Remove initialize_from_frozen_model flag and support code --- DeepSpeech.py | 29 ----------------- README.md | 10 +++--- bin/run-tc-ldc93s1_checkpoint.sh | 31 +++++++++++++++++++ bin/run-tc-ldc93s1_frozen.sh | 23 -------------- bin/run-tc-ldc93s1_new.sh | 7 +++-- ..._upstream_frozen-linux-amd64-py36m-opt.yml | 13 -------- tc-tests-utils.sh | 5 --- tc-train-tests.sh | 9 ++---- 8 files changed, 43 insertions(+), 84 deletions(-) create mode 100755 bin/run-tc-ldc93s1_checkpoint.sh delete mode 100755 bin/run-tc-ldc93s1_frozen.sh delete mode 100644 taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml diff --git a/DeepSpeech.py b/DeepSpeech.py index 8711c9d5..efda6a0b 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -168,10 +168,6 @@ def create_flags(): tf.app.flags.DEFINE_string ('one_shot_infer', '', 'one-shot inference mode: specify a wav file and the script will load the checkpoint and perform inference on it. Disables training, testing and exporting.') - # Initialize from frozen model - - tf.app.flags.DEFINE_string ('initialize_from_frozen_model', '', 'path to frozen model to initialize from. This behaves like a checkpoint, loading the weights from the frozen model and starting training with those weights. The optimizer parameters aren\'t restored, so remember to adjust the learning rate.') - FLAGS = tf.app.flags.FLAGS def initialize_globals(): @@ -1579,26 +1575,6 @@ def train(server=None): saver = tf.train.Saver(max_to_keep=FLAGS.max_to_keep) hooks.append(tf.train.CheckpointSaverHook(checkpoint_dir=FLAGS.checkpoint_dir, save_secs=FLAGS.checkpoint_secs, saver=saver)) - if len(FLAGS.initialize_from_frozen_model) > 0: - with tf.gfile.FastGFile(FLAGS.initialize_from_frozen_model, 'rb') as fin: - graph_def = tf.GraphDef() - graph_def.ParseFromString(fin.read()) - - var_names = [v.name for v in tf.trainable_variables()] - var_tensors = tf.import_graph_def(graph_def, return_elements=var_names) - - # build a { var_name: var_tensor } dict - var_tensors = dict(zip(var_names, var_tensors)) - - training_graph = tf.get_default_graph() - - assign_ops = [] - for name, restored_tensor in var_tensors.items(): - training_tensor = training_graph.get_tensor_by_name(name) - assign_ops.append(tf.assign(training_tensor, restored_tensor)) - - init_from_frozen_model_op = tf.group(*assign_ops) - no_dropout_feed_dict = { dropout_rates[0]: 0., dropout_rates[1]: 0., @@ -1661,11 +1637,6 @@ def train(server=None): config=session_config) as session: tf.get_default_graph().finalize() - if len(FLAGS.initialize_from_frozen_model) > 0: - log_info('Initializing from frozen model: {}'.format(FLAGS.initialize_from_frozen_model)) - model_feeder.set_data_set(no_dropout_feed_dict, model_feeder.train) - session.run(init_from_frozen_model_op, feed_dict=no_dropout_feed_dict) - try: if is_chief: # Retrieving global_step from the (potentially restored) model diff --git a/README.md b/README.md index 8ba68947..10f41e99 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech - [Checkpointing](#checkpointing) - [Exporting a model for inference](#exporting-a-model-for-inference) - [Distributed computing across more than one machine](#distributed-training-across-more-than-one-machine) - - [Continuing training from a frozen graph](#continuing-training-from-a-frozen-graph) + - [Continuing training from a release model](#continuing-training-from-a-release-model) - [Code documentation](#code-documentation) - [Contact/Getting Help](#contactgetting-help) @@ -353,18 +353,18 @@ $ run-cluster.sh 1:2:1 --epoch 10 Be aware that for the help example to be able to run, you need at least two `CUDA` capable GPUs (2 workers times 1 GPU). The script utilizes environment variable `CUDA_VISIBLE_DEVICES` for `DeepSpeech.py` to see only the provided number of GPUs per worker. The script is meant to be a template for your own distributed computing instrumentation. Just modify the startup code for the different servers (workers and parameter servers) accordingly. You could use SSH or something similar for running them on your remote hosts. -### Continuing training from a frozen graph +### Continuing training from a release model -If you'd like to use one of the pre-trained models released by Mozilla to bootstrap your training process (transfer learning, fine tuning), you can do so by using the `--initialize_from_frozen_model` flag in `DeepSpeech.py`. For best results, make sure you're passing an empty `--checkpoint_dir` when resuming from a frozen model. +If you'd like to use one of the pre-trained models released by Mozilla to bootstrap your training process (transfer learning, fine tuning), you can do so by using the `--checkpoint_dir` flag in `DeepSpeech.py`. Specify the path where you downloaded the checkpoint from the release, and training will resume from the pre-trained model. For example, if you want to fine tune the entire graph using your own data in `my-train.csv`, `my-dev.csv` and `my-test.csv`, for three epochs, you can something like the following, tuning the hyperparameters as needed: ```bash mkdir fine_tuning_checkpoints -python3 DeepSpeech.py --n_hidden 2048 --initialize_from_frozen_model path/to/model/output_graph.pb --checkpoint_dir fine_tuning_checkpoints --epoch 3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001 +python3 DeepSpeech.py --n_hidden 2048 --checkpoint_dir path/to/checkpoint/folder --epoch -3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001 ``` -Note: the released models were trained with `--n_hidden 2048`, so you need to use that same value when initializing from the release models. +Note: the released models were trained with `--n_hidden 2048`, so you need to use that same value when initializing from the release models. Note as well the use of a negative epoch count -3 (meaning 3 more epochs) since the checkpoint you're loading from was already trained for several epochs. ## Code documentation diff --git a/bin/run-tc-ldc93s1_checkpoint.sh b/bin/run-tc-ldc93s1_checkpoint.sh new file mode 100755 index 00000000..6ebb597d --- /dev/null +++ b/bin/run-tc-ldc93s1_checkpoint.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -xe + +ldc93s1_dir="./data/ldc93s1-tc" +ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv" + +epoch_count=$1 + +if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then + echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}." + python -u bin/import_ldc93s1.py ${ldc93s1_dir} +fi; + +python -u DeepSpeech.py --noshow_progressbar \ + --train_files ${ldc93s1_csv} --train_batch_size 1 \ + --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ + --test_files ${ldc93s1_csv} --test_batch_size 1 \ + --n_hidden 494 --epoch -1 --random_seed 4567 --default_stddev 0.046875 \ + --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' \ + --learning_rate 0.001 --dropout_rate 0.05 \ + --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \ + --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ + --lm_trie_path 'data/smoke_test/vocab.trie' | tee /tmp/resume.log + +if ! grep "Training of Epoch $epoch_count" /tmp/resume.log; then + echo "Did not resume training from checkpoint" + exit 1 +else + exit 0 +fi diff --git a/bin/run-tc-ldc93s1_frozen.sh b/bin/run-tc-ldc93s1_frozen.sh deleted file mode 100755 index 0ea633b6..00000000 --- a/bin/run-tc-ldc93s1_frozen.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -set -xe - -ldc93s1_dir="./data/ldc93s1-tc" -ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv" - -if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then - echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}." - python -u bin/import_ldc93s1.py ${ldc93s1_dir} -fi; - -python -u DeepSpeech.py \ - --train_files ${ldc93s1_csv} --train_batch_size 1 \ - --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ - --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch 1 --random_seed 4567 --default_stddev 0.046875 \ - --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \ - --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/train' \ - --nouse_seq_length --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \ - --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ - --lm_trie_path 'data/smoke_test/vocab.trie' \ - --initialize_from_frozen_model '/tmp/frozen_model.pb' diff --git a/bin/run-tc-ldc93s1_new.sh b/bin/run-tc-ldc93s1_new.sh index 1aabd98d..a5d12cb9 100755 --- a/bin/run-tc-ldc93s1_new.sh +++ b/bin/run-tc-ldc93s1_new.sh @@ -5,6 +5,8 @@ set -xe ldc93s1_dir="./data/ldc93s1-tc" ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv" +epoch_count=$1 + if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}." python -u bin/import_ldc93s1.py ${ldc93s1_dir} @@ -14,8 +16,9 @@ python -u DeepSpeech.py \ --train_files ${ldc93s1_csv} --train_batch_size 1 \ --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch 105 --random_seed 4567 --default_stddev 0.046875 \ - --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \ + --n_hidden 494 --epoch $epoch_count --random_seed 4567 \ + --default_stddev 0.046875 --max_to_keep 1 \ + --checkpoint_dir '/tmp/ckpt' \ --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/train' \ --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ diff --git a/taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml b/taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml deleted file mode 100644 index ae81cc3d..00000000 --- a/taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml +++ /dev/null @@ -1,13 +0,0 @@ -build: - template_file: test-linux-opt-base.tyml - dependencies: - - "linux-amd64-ctc-opt" - - "test-training_upstream-linux-amd64-py27mu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} - args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m frozen" - metadata: - name: "DeepSpeech Linux AMD64 CPU upstream training frozen Py3.6" - description: "Training a DeepSpeech LDC93S1 model from frozen model file for Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 525e3683..e4999410 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -265,11 +265,6 @@ download_data() cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/smoke_test/vocab.trie ${TASKCLUSTER_TMP_DIR}/trie } -download_for_frozen() -{ - wget -O "${TASKCLUSTER_TMP_DIR}/frozen_model.pb" "${DEEPSPEECH_TEST_MODEL}" -} - download_material() { target_dir=$1 diff --git a/tc-train-tests.sh b/tc-train-tests.sh index 121cca10..7ce7d054 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -6,7 +6,6 @@ source $(dirname "$0")/tc-tests-utils.sh pyver_full=$1 ds=$2 -frozen=$2 if [ -z "${pyver_full}" ]; then echo "No python version given, aborting." @@ -62,12 +61,8 @@ else fi; pushd ${HOME}/DeepSpeech/ds/ - if [ "${frozen}" = "frozen" ]; then - download_for_frozen - time ./bin/run-tc-ldc93s1_frozen.sh - else - time ./bin/run-tc-ldc93s1_new.sh - fi; + time ./bin/run-tc-ldc93s1_new.sh 105 + time ./bin/run-tc-ldc93s1_checkpoint.sh 105 popd deactivate From 4be9364c364ce6bbe83847530da4d2e0265d88a1 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 24 Oct 2018 11:16:52 +0200 Subject: [PATCH 14/18] Force HOMEBREW_NO_AUTO_UPDATE=1 to avoid magic autoupdate of brew --- native_client/definitions.mk | 3 ++- native_client/javascript/binding.gyp | 15 +++++++++++++++ taskcluster/darwin-opt-base.tyml | 4 +++- taskcluster/test-darwin-opt-base.tyml | 4 +++- tc-brew-tests.sh | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 7796da9d..4ed675b9 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -65,7 +65,8 @@ LDFLAGS_NEEDED := -Wl,--no-as-needed LDFLAGS_RPATH := -Wl,-rpath,\$$ORIGIN endif ifeq ($(OS),Darwin) -LDFLAGS_NEEDED := +CXXFLAGS += -stdlib=libc++ -mmacosx-version-min=10.10 +LDFLAGS_NEEDED := -stdlib=libc++ -mmacosx-version-min=10.10 LDFLAGS_RPATH := -Wl,-rpath,@executable_path endif diff --git a/native_client/javascript/binding.gyp b/native_client/javascript/binding.gyp index d85dee28..789e4a91 100644 --- a/native_client/javascript/binding.gyp +++ b/native_client/javascript/binding.gyp @@ -8,6 +8,21 @@ ], "include_dirs": [ "../" + ], + "conditions": [ + [ "OS=='mac'", { + "xcode_settings": { + "OTHER_CXXFLAGS": [ + "-stdlib=libc++", + "-mmacosx-version-min=10.10" + ], + "OTHER_LDFLAGS": [ + "-stdlib=libc++", + "-mmacosx-version-min=10.10" + ] + } + } + ] ] }, { diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index cd398974..5157302c 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -87,6 +87,7 @@ payload: export LC_ALL=C && export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" && export MACOSX_DEPLOYMENT_TARGET=10.10 && + export HOMEBREW_NO_AUTO_UPDATE=1 && env && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && @@ -97,9 +98,10 @@ payload: ${swig.patch_nodejs.osx} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ; + export TASKCLUSTER_TASK_EXIT_CODE=$? && mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && - cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE artifacts: - type: "directory" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 039f54bb..337fc65e 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -59,6 +59,7 @@ then: export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" && export LC_ALL=C && export MACOSX_DEPLOYMENT_TARGET=10.10 && + export HOMEBREW_NO_AUTO_UPDATE=1 && export PIP_DEFAULT_TIMEOUT=60 && env && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && @@ -66,9 +67,10 @@ then: cd $TASKCLUSTER_TASK_DIR && source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-brew-tests.sh && ${extraSystemSetup} && /bin/bash ${build.args.tests_cmdline} ; + export TASKCLUSTER_TASK_EXIT_CODE=$? && mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && - cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ + cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE mounts: - cacheName: deepspeech-homebrew-bin diff --git a/tc-brew-tests.sh b/tc-brew-tests.sh index fa51d5cb..1a861da5 100644 --- a/tc-brew-tests.sh +++ b/tc-brew-tests.sh @@ -37,7 +37,7 @@ install_local_homebrew() mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}" mkdir -p "${HOMEBREW_CACHE}" - curl -L https://github.com/Homebrew/brew/tarball/1.7.7 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}" + curl -L https://github.com/Homebrew/brew/tarball/1.8.0 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}" export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then From ae5303e130acaa41e4ee81b17a197e6a119110bb Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 25 Oct 2018 10:08:20 +0200 Subject: [PATCH 15/18] Update TensorFlow r1.11 --- taskcluster/darwin-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-ctc-opt.yml | 4 ++-- taskcluster/linux-amd64-gpu-opt.yml | 4 ++-- taskcluster/linux-arm64-cpu-opt.yml | 4 ++-- taskcluster/linux-rpi3-cpu-opt.yml | 4 ++-- taskcluster/node-package.yml | 4 ++-- taskcluster/test-armbian-opt-base.tyml | 2 +- taskcluster/test-darwin-opt-base.tyml | 2 +- taskcluster/test-linux-opt-base.tyml | 2 +- taskcluster/test-raspbian-opt-base.tyml | 2 +- taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 99ed3ea7..2426c7c9 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -6,8 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.osx/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 794b0c4d..95c141a1 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -14,8 +14,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index 04a337f2..535fcfbd 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,8 +4,8 @@ build: - "pull_request.synchronize" - "pull_request.reopened" template_file: linux-opt-base.tyml - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/summarize_graph" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 8b3e1f08..22ba8b2c 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -12,8 +12,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.gpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.gpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.gpu/artifacts/public/summarize_graph" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 9326f3ef..49d82bcc 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm64/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.arm64/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 86998800..0c78780e 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index f21709ff..d937bd41 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -16,8 +16,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index 4feb0f3b..08b85c3d 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-11-gbee8254" command: - "/bin/bash" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 337fc65e..e889ad40 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -41,7 +41,7 @@ then: DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-11-gbee8254" command: - - "/bin/bash" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 79a162bf..37c3cf85 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -44,7 +44,7 @@ then: DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-11-gbee8254" command: - "/bin/bash" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 71374453..f5d7484f 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-9-g97d851f" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.11.0-11-gbee8254" command: - "/bin/bash" diff --git a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml index 8b910c5e..ef98a1e4 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu" - convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.97d851f04e01696e51bc59b9bb4753fd22b3b25e.cpu/artifacts/public/convert_graphdef_memmapped_format" + convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.11.bee825492fcf830bd65a024bf859cbfc218e1473.cpu/artifacts/public/convert_graphdef_memmapped_format" metadata: name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version" From fc9b6e3c2cd12e85273c1a529b89a8d94367c6bb Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 25 Oct 2018 13:28:32 +0200 Subject: [PATCH 16/18] Export trained model before testing with checkpoint --- tc-train-tests.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tc-train-tests.sh b/tc-train-tests.sh index 7ce7d054..b6de6d9c 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -62,12 +62,8 @@ fi; pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1_new.sh 105 - time ./bin/run-tc-ldc93s1_checkpoint.sh 105 popd -deactivate -pyenv uninstall --force ${PYENV_NAME} - cp /tmp/train/output_graph.pb ${TASKCLUSTER_ARTIFACTS} if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then @@ -77,3 +73,10 @@ if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then /tmp/${convert_graphdef} --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm cp /tmp/train/output_graph.pbmm ${TASKCLUSTER_ARTIFACTS} fi; + +pushd ${HOME}/DeepSpeech/ds/ + time ./bin/run-tc-ldc93s1_checkpoint.sh 105 +popd + +deactivate +pyenv uninstall --force ${PYENV_NAME} From b0bf13eec35ee6faad88cbee29bb9c237f29cfd0 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 25 Oct 2018 15:51:18 +0200 Subject: [PATCH 17/18] Update TensorFlow master --- taskcluster/darwin-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-cpu-opt.yml | 4 ++-- taskcluster/linux-amd64-ctc-opt.yml | 4 ++-- taskcluster/linux-amd64-cuda-opt.yml | 4 ++-- taskcluster/linux-arm64-cpu-opt.yml | 4 ++-- taskcluster/linux-rpi3-cpu-opt.yml | 4 ++-- taskcluster/node-package.yml | 4 ++-- taskcluster/test-armbian-opt-base.tyml | 2 +- taskcluster/test-darwin-opt-base.tyml | 2 +- taskcluster/test-linux-opt-base.tyml | 2 +- taskcluster/test-raspbian-opt-base.tyml | 2 +- taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 1bfc3c29..5cd89616 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -6,8 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.osx/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.osx/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.osx/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.osx/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index d7c4f11c..1fede7e0 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -14,8 +14,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index 26aa4d1c..9a25a115 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,8 +4,8 @@ build: - "pull_request.synchronize" - "pull_request.reopened" template_file: linux-opt-base.tyml - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-cuda-opt.yml b/taskcluster/linux-amd64-cuda-opt.yml index bda53483..616ba526 100644 --- a/taskcluster/linux-amd64-cuda-opt.yml +++ b/taskcluster/linux-amd64-cuda-opt.yml @@ -12,8 +12,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cuda/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cuda/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cuda/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cuda/artifacts/public/summarize_graph" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 1fe1ad63..8a57fa55 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.arm64/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.arm64/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 25990dd2..fd174be3 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,8 +4,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.arm/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.arm/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > diff --git a/taskcluster/node-package.yml b/taskcluster/node-package.yml index bd94d011..b724821c 100644 --- a/taskcluster/node-package.yml +++ b/taskcluster/node-package.yml @@ -16,8 +16,8 @@ build: system_config: > ${swig.patch_nodejs.linux} - tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz" - summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph" + tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz" + summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph" scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index 8e80eb5e..1c674b8d 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736" command: - "/bin/bash" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index a9faad34..cc648718 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -41,7 +41,7 @@ then: DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736" command: - - "/bin/bash" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 8f9259a2..262b293a 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -44,7 +44,7 @@ then: DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736" command: - "/bin/bash" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index ca3a08a7..7c6062f8 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -44,7 +44,7 @@ then: PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels - EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde" + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736" command: - "/bin/bash" diff --git a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml index 4c422c6e..1f2bc3a0 100644 --- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml +++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml @@ -7,7 +7,7 @@ build: apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu" - convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/convert_graphdef_memmapped_format" + convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/convert_graphdef_memmapped_format" metadata: name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version" From 0445cb6ab36a1c84b32c3aea03ecf5a0463754af Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 25 Oct 2018 15:57:05 +0200 Subject: [PATCH 18/18] Fix bogus merge --- taskcluster/test-darwin-opt-base.tyml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index cc648718..ae263397 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -67,16 +67,10 @@ then: cd $TASKCLUSTER_TASK_DIR && source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-brew-tests.sh && ${extraSystemSetup} && /bin/bash ${build.args.tests_cmdline} ; -<<<<<<< HEAD - mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && - mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && - cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ -======= export TASKCLUSTER_TASK_EXIT_CODE=$? && mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ && mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ && cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE ->>>>>>> upstream/master mounts: - cacheName: deepspeech-homebrew-bin