Merge pull request #1680 from lissyx/update-tf-master

Update tf master
2025-10-26 11:19:39 +00:00 · 2018-10-25 17:56:55 +02:00 · 2018-10-25 17:56:55 +02:00 · f80272eb03
commit f80272eb03
parent 80e91900cd 0445cb6ab3
35 changed files with 188 additions and 168 deletions
--- a/DeepSpeech.py
+++ b/DeepSpeech.py
@ -162,16 +162,12 @@ def create_flags():
    tf.app.flags.DEFINE_string  ('lm_trie_path',         'data/lm/trie', 'path to the language model trie file created with native_client/generate_trie')
    tf.app.flags.DEFINE_integer ('beam_width',        1024,       'beam width used in the CTC decoder when building candidate transcriptions')
    tf.app.flags.DEFINE_float   ('lm_weight',         1.50,       'the alpha hyperparameter of the CTC decoder. Language Model weight.')
-    tf.app.flags.DEFINE_float   ('valid_word_count_weight', 2.25, 'valid word insertion weight. This is used to lessen the word insertion penalty when the inserted word is part of the vocabulary.')
+    tf.app.flags.DEFINE_float   ('valid_word_count_weight', 2.10, 'valid word insertion weight. This is used to lessen the word insertion penalty when the inserted word is part of the vocabulary.')

    # Inference mode

    tf.app.flags.DEFINE_string  ('one_shot_infer',       '',       'one-shot inference mode: specify a wav file and the script will load the checkpoint and perform inference on it. Disables training, testing and exporting.')

-    # Initialize from frozen model
-
-    tf.app.flags.DEFINE_string  ('initialize_from_frozen_model', '', 'path to frozen model to initialize from. This behaves like a checkpoint, loading the weights from the frozen model and starting training with those weights. The optimizer parameters aren\'t restored, so remember to adjust the learning rate.')
-
 FLAGS = tf.app.flags.FLAGS

 def initialize_globals():
@ -870,15 +866,15 @@ def new_id():
    return id_counter

 class Sample(object):
-    def __init__(self, src, res, loss, mean_edit_distance, sample_wer):
-        '''Represents one item of a WER report.
+    '''Represents one item of a WER report.

-        Args:
-            src (str): source text
-            res (str): resulting text
-            loss (float): computed loss of this item
-            mean_edit_distance (float): computed mean edit distance of this item
-        '''
+    Args:
+        src (str): source text
+        res (str): resulting text
+        loss (float): computed loss of this item
+        mean_edit_distance (float): computed mean edit distance of this item
+    '''
+    def __init__(self, src, res, loss, mean_edit_distance, sample_wer):
        self.src = src
        self.res = res
        self.loss = loss
@ -889,16 +885,16 @@ class Sample(object):
        return 'WER: %f, loss: %f, mean edit distance: %f\n - src: "%s"\n - res: "%s"' % (self.wer, self.loss, self.mean_edit_distance, self.src, self.res)

 class WorkerJob(object):
-    def __init__(self, epoch_id, index, set_name, steps, report):
-        '''Represents a job that should be executed by a worker.
+    '''Represents a job that should be executed by a worker.

-        Args:
-            epoch_id (int): the ID of the 'parent' epoch
-            index (int): the epoch index of the 'parent' epoch
-            set_name (str): the name of the data-set - one of 'train', 'dev', 'test'
-            steps (int): the number of `session.run` calls
-            report (bool): if this job should produce a WER report
-        '''
+    Args:
+        epoch_id (int): the ID of the 'parent' epoch
+        index (int): the epoch index of the 'parent' epoch
+        set_name (str): the name of the data-set - one of 'train', 'dev', 'test'
+        steps (int): the number of `session.run` calls
+        report (bool): if this job should produce a WER report
+    '''
+    def __init__(self, epoch_id, index, set_name, steps, report):
        self.id = new_id()
        self.epoch_id = epoch_id
        self.index = index
@ -1070,6 +1066,12 @@ class Epoch(object):


 class TrainingCoordinator(object):
+    ''' Central training coordination class.
+    Used for distributing jobs among workers of a cluster.
+    Instantiated on all workers, calls of non-chief workers will transparently
+    HTTP-forwarded to the chief worker instance.
+    '''
+
    class TrainingCoordinationHandler(BaseHTTPServer.BaseHTTPRequestHandler):
        '''Handles HTTP requests from remote workers to the Training Coordinator.
        '''
@ -1114,13 +1116,7 @@ class TrainingCoordinator(object):
            '''
            return

-
    def __init__(self):
-        ''' Central training coordination class.
-        Used for distributing jobs among workers of a cluster.
-        Instantiated on all workers, calls of non-chief workers will transparently
-        HTTP-forwarded to the chief worker instance.
-        '''
        self._init()
        self._lock = Lock()
        self.started = False
@ -1579,26 +1575,6 @@ def train(server=None):
        saver = tf.train.Saver(max_to_keep=FLAGS.max_to_keep)
        hooks.append(tf.train.CheckpointSaverHook(checkpoint_dir=FLAGS.checkpoint_dir, save_secs=FLAGS.checkpoint_secs, saver=saver))

-    if len(FLAGS.initialize_from_frozen_model) > 0:
-        with tf.gfile.FastGFile(FLAGS.initialize_from_frozen_model, 'rb') as fin:
-            graph_def = tf.GraphDef()
-            graph_def.ParseFromString(fin.read())
-
-        var_names = [v.name for v in tf.trainable_variables()]
-        var_tensors = tf.import_graph_def(graph_def, return_elements=var_names)
-
-        # build a { var_name: var_tensor } dict
-        var_tensors = dict(zip(var_names, var_tensors))
-
-        training_graph = tf.get_default_graph()
-
-        assign_ops = []
-        for name, restored_tensor in var_tensors.items():
-            training_tensor = training_graph.get_tensor_by_name(name)
-            assign_ops.append(tf.assign(training_tensor, restored_tensor))
-
-        init_from_frozen_model_op = tf.group(*assign_ops)
-
    no_dropout_feed_dict = {
        dropout_rates[0]: 0.,
        dropout_rates[1]: 0.,
@ -1661,11 +1637,6 @@ def train(server=None):
                                               config=session_config) as session:
            tf.get_default_graph().finalize()

-            if len(FLAGS.initialize_from_frozen_model) > 0:
-                log_info('Initializing from frozen model: {}'.format(FLAGS.initialize_from_frozen_model))
-                model_feeder.set_data_set(no_dropout_feed_dict, model_feeder.train)
-                session.run(init_from_frozen_model_op, feed_dict=no_dropout_feed_dict)
-
            try:
                if is_chief:
                    # Retrieving global_step from the (potentially restored) model
--- a/README.md
+++ b/README.md
@ -46,7 +46,7 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech
  - [Checkpointing](#checkpointing)
  - [Exporting a model for inference](#exporting-a-model-for-inference)
  - [Distributed computing across more than one machine](#distributed-training-across-more-than-one-machine)
-  - [Continuing training from a frozen graph](#continuing-training-from-a-frozen-graph)
+  - [Continuing training from a release model](#continuing-training-from-a-release-model)
 - [Code documentation](#code-documentation)
 - [Contact/Getting Help](#contactgetting-help)

@ -69,7 +69,7 @@ git clone https://github.com/mozilla/DeepSpeech
 If you want to use the pre-trained English model for performing speech-to-text, you can download it (along with other important inference material) from the [DeepSpeech releases page](https://github.com/mozilla/DeepSpeech/releases). Alternatively, you can run the following command to download and unzip the files in your current directory:

 ```bash
-wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.2.0/deepspeech-0.2.0-models.tar.gz | tar xvfz -
+wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.3.0/deepspeech-0.3.0-models.tar.gz | tar xvfz -
 ```

 ## Using the model
@ -353,18 +353,18 @@ $ run-cluster.sh 1:2:1 --epoch 10
 Be aware that for the help example to be able to run, you need at least two `CUDA` capable GPUs (2 workers times 1 GPU). The script utilizes environment variable `CUDA_VISIBLE_DEVICES` for `DeepSpeech.py` to see only the provided number of GPUs per worker.
 The script is meant to be a template for your own distributed computing instrumentation. Just modify the startup code for the different servers (workers and parameter servers) accordingly. You could use SSH or something similar for running them on your remote hosts.

-### Continuing training from a frozen graph
+### Continuing training from a release model

-If you'd like to use one of the pre-trained models released by Mozilla to bootstrap your training process (transfer learning, fine tuning), you can do so by using the `--initialize_from_frozen_model` flag in `DeepSpeech.py`. For best results, make sure you're passing an empty `--checkpoint_dir` when resuming from a frozen model.
+If you'd like to use one of the pre-trained models released by Mozilla to bootstrap your training process (transfer learning, fine tuning), you can do so by using the `--checkpoint_dir` flag in `DeepSpeech.py`. Specify the path where you downloaded the checkpoint from the release, and training will resume from the pre-trained model.

 For example, if you want to fine tune the entire graph using your own data in `my-train.csv`, `my-dev.csv` and `my-test.csv`, for three epochs, you can something like the following, tuning the hyperparameters as needed:

 ```bash
 mkdir fine_tuning_checkpoints
-python3 DeepSpeech.py --n_hidden 2048 --initialize_from_frozen_model path/to/model/output_graph.pb --checkpoint_dir fine_tuning_checkpoints --epoch 3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001
+python3 DeepSpeech.py --n_hidden 2048 --checkpoint_dir path/to/checkpoint/folder --epoch -3 --train_files my-train.csv --dev_files my-dev.csv --test_files my_dev.csv --learning_rate 0.0001
 ```

-Note: the released models were trained with `--n_hidden 2048`, so you need to use that same value when initializing from the release models.
+Note: the released models were trained with `--n_hidden 2048`, so you need to use that same value when initializing from the release models. Note as well the use of a negative epoch count -3 (meaning 3 more epochs) since the checkpoint you're loading from was already trained for several epochs.

 ## Code documentation

--- a/2
+++ b/2
@ -1 +1 @@
-0.3.0-alpha.0
+0.3.0
--- a/bin/import_cv.py
+++ b/bin/import_cv.py
@ -10,6 +10,7 @@ sys.path.insert(1, os.path.join(sys.path[0], '..'))
 import csv
 import tarfile
 import subprocess
+import progressbar

 from glob import glob
 from os import path
--- a/bin/import_ts.py
+++ b/bin/import_ts.py
@ -3,6 +3,7 @@ from __future__ import absolute_import, division, print_function

 # Make sure we can import stuff from util/
 # This script needs to be run from the root of the DeepSpeech repository
+import argparse
 import os
 import re
 import sys
@ -11,6 +12,7 @@ import sys
 sys.path.insert(1, os.path.join(sys.path[0], '..'))

 import csv
+import unidecode
 import zipfile

 from os import path
@ -25,7 +27,7 @@ ARCHIVE_DIR_NAME = 'ts_' + ARCHIVE_NAME
 ARCHIVE_URL = 'https://s3.eu-west-3.amazonaws.com/audiocorp/releases/' + ARCHIVE_NAME + '.zip'


-def _download_and_preprocess_data(target_dir):
+def _download_and_preprocess_data(target_dir, english_compatible=False):
    # Making path absolute
    target_dir = path.abspath(target_dir)
    # Conditionally download data
@ -33,7 +35,8 @@ def _download_and_preprocess_data(target_dir):
    # Conditionally extract archive data
    _maybe_extract(target_dir, ARCHIVE_DIR_NAME, archive_path)
    # Conditionally convert TrainingSpeech data to DeepSpeech CSVs and wav
-    _maybe_convert_sets(target_dir, ARCHIVE_DIR_NAME)
+    _maybe_convert_sets(target_dir, ARCHIVE_DIR_NAME, english_compatible=english_compatible)
+

 def _maybe_extract(target_dir, extracted_data, archive_path):
    # If target_dir/extracted_data does not exist, extract archive in target_dir
@ -47,7 +50,8 @@ def _maybe_extract(target_dir, extracted_data, archive_path):
    else:
        print('Found directory "%s" - not extracting it from archive.' % archive_path)

-def _maybe_convert_sets(target_dir, extracted_data):
+
+def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False):
    extracted_dir = path.join(target_dir, extracted_data)
    # override existing CSV with normalized one
    target_csv_template = os.path.join(target_dir, 'ts_' + ARCHIVE_NAME + '_{}.csv')
@ -70,7 +74,7 @@ def _maybe_convert_sets(target_dir, extracted_data):
                test_writer.writeheader()

                for i, item in enumerate(data):
-                    transcript = validate_label(cleanup_transcript(item['text']))
+                    transcript = validate_label(cleanup_transcript(item['text'], english_compatible=english_compatible))
                    if not transcript:
                        continue
                    wav_filename = os.path.join(target_dir, extracted_data, item['path'])
@ -92,12 +96,22 @@ PUNCTUATIONS_REG = re.compile(r"[°\-,;!?.()\[\]*…—]")
 MULTIPLE_SPACES_REG = re.compile(r'\s{2,}')


-def cleanup_transcript(text):
+def cleanup_transcript(text, english_compatible=False):
    text = text.replace('’', "'").replace('\u00A0', ' ')
    text = PUNCTUATIONS_REG.sub(' ', text)
    text = MULTIPLE_SPACES_REG.sub(' ', text)
+    if english_compatible:
+        text = unidecode.unidecode(text)
    return text.strip().lower()


+def handle_args():
+    parser = argparse.ArgumentParser(description='Importer for TrainingSpeech dataset.')
+    parser.add_argument(dest='target_dir')
+    parser.add_argument('--english-compatible', action='store_true', dest='english_compatible', help='Remove diactrics and other non-ascii chars.')
+    return parser.parse_args()
+
+
 if __name__ == "__main__":
-    _download_and_preprocess_data(sys.argv[1])
+    cli_args = handle_args()
+    _download_and_preprocess_data(cli_args.target_dir, cli_args.english_compatible)
--- a/bin/run-tc-ldc93s1_checkpoint.sh
+++ b/bin/run-tc-ldc93s1_checkpoint.sh
@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -xe
+
+ldc93s1_dir="./data/ldc93s1-tc"
+ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv"
+
+epoch_count=$1
+
+if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then
+    echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}."
+    python -u bin/import_ldc93s1.py ${ldc93s1_dir}
+fi;
+
+python -u DeepSpeech.py --noshow_progressbar \
+  --train_files ${ldc93s1_csv} --train_batch_size 1 \
+  --dev_files ${ldc93s1_csv} --dev_batch_size 1 \
+  --test_files ${ldc93s1_csv} --test_batch_size 1 \
+  --n_hidden 494 --epoch -1 --random_seed 4567 --default_stddev 0.046875 \
+  --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' \
+  --learning_rate 0.001 --dropout_rate 0.05 \
+  --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \
+  --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \
+  --lm_trie_path 'data/smoke_test/vocab.trie' | tee /tmp/resume.log
+
+if ! grep "Training of Epoch $epoch_count" /tmp/resume.log; then
+  echo "Did not resume training from checkpoint"
+  exit 1
+else
+  exit 0
+fi
--- a/bin/run-tc-ldc93s1_frozen.sh
+++ b/bin/run-tc-ldc93s1_frozen.sh
@ -1,23 +0,0 @@
-#!/bin/sh
-
-set -xe
-
-ldc93s1_dir="./data/ldc93s1-tc"
-ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv"
-
-if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then
-    echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}."
-    python -u bin/import_ldc93s1.py ${ldc93s1_dir}
-fi;
-
-python -u DeepSpeech.py \
-  --train_files ${ldc93s1_csv} --train_batch_size 1 \
-  --dev_files ${ldc93s1_csv} --dev_batch_size 1 \
-  --test_files ${ldc93s1_csv} --test_batch_size 1 \
-  --n_hidden 494 --epoch 1 --random_seed 4567 --default_stddev 0.046875 \
-  --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \
-  --learning_rate 0.001 --dropout_rate 0.05  --export_dir '/tmp/train' \
-  --nouse_seq_length --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \
-  --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \
-  --lm_trie_path 'data/smoke_test/vocab.trie' \
-  --initialize_from_frozen_model '/tmp/frozen_model.pb'
--- a/bin/run-tc-ldc93s1_new.sh
+++ b/bin/run-tc-ldc93s1_new.sh
@ -5,6 +5,8 @@ set -xe
 ldc93s1_dir="./data/ldc93s1-tc"
 ldc93s1_csv="${ldc93s1_dir}/ldc93s1.csv"

+epoch_count=$1
+
 if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then
    echo "Downloading and preprocessing LDC93S1 example data, saving in ${ldc93s1_dir}."
    python -u bin/import_ldc93s1.py ${ldc93s1_dir}
@ -14,8 +16,9 @@ python -u DeepSpeech.py \
  --train_files ${ldc93s1_csv} --train_batch_size 1 \
  --dev_files ${ldc93s1_csv} --dev_batch_size 1 \
  --test_files ${ldc93s1_csv} --test_batch_size 1 \
-  --n_hidden 494 --epoch 85 --random_seed 4567 --default_stddev 0.046875 \
-  --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \
+  --n_hidden 494 --epoch $epoch_count --random_seed 4567 \
+  --default_stddev 0.046875 --max_to_keep 1 \
+  --checkpoint_dir '/tmp/ckpt' \
  --learning_rate 0.001 --dropout_rate 0.05  --export_dir '/tmp/train' \
  --decoder_library_path '/tmp/ds/libctc_decoder_with_kenlm.so' \
  --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \
--- a/evaluate.py
+++ b/evaluate.py
@ -99,12 +99,31 @@ def main(_):

    # sort examples by length, improves packing of batches and timesteps
    test_data = preprocess(
-        FLAGS.test_files,
+        FLAGS.test_files.split(','),
        FLAGS.test_batch_size,
-        hdf5_dest_path=FLAGS.hdf5_test_set).sort_values(
+        alphabet=alphabet,
+        numcep=N_FEATURES,
+        numcontext=N_CONTEXT,
+        hdf5_cache_path=FLAGS.hdf5_test_set).sort_values(
        by="features_len",
        ascending=False)

+    def create_windows(features):
+        num_strides = len(features) - (N_CONTEXT * 2)
+
+        # Create a view into the array with overlapping strides of size
+        # numcontext (past) + 1 (present) + numcontext (future)
+        window_size = 2*N_CONTEXT+1
+        features = np.lib.stride_tricks.as_strided(
+            features,
+            (num_strides, window_size, N_FEATURES),
+            (features.strides[0], features.strides[0], features.strides[1]),
+            writeable=False)
+
+        return features
+
+    test_data['features'] = test_data['features'].apply(create_windows)
+
    with tf.Session() as session:
        inputs, outputs = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=N_STEPS)

@ -158,7 +177,7 @@ def main(_):

            logits = np.empty([0, FLAGS.test_batch_size, alphabet.size() + 1])
            for i in range(0, batch_features.shape[1], N_STEPS):
-                chunk_features = batch_features[:, i:i + N_STEPS, :]
+                chunk_features = batch_features[:, i:i + N_STEPS, :, :]
                chunk_features_len = np.minimum(
                    batch_features_len, full_step_len)

@ -168,6 +187,7 @@ def main(_):
                    chunk_features = np.pad(chunk_features,
                                            ((0, 0),
                                             (0, FLAGS.n_steps - steps_in_chunk),
+                                             (0, 0),
                                             (0, 0)),
                                            mode='constant',
                                            constant_values=0)
--- a/examples/vad_transcriber/wavTranscriber.py
+++ b/examples/vad_transcriber/wavTranscriber.py
@ -19,8 +19,8 @@ def load_model(models, alphabet, lm, trie):
    N_FEATURES = 26
    N_CONTEXT = 9
    BEAM_WIDTH = 500
-    LM_WEIGHT = 1.75
-    VALID_WORD_COUNT_WEIGHT = 1.00
+    LM_WEIGHT = 1.50
+    VALID_WORD_COUNT_WEIGHT = 2.10

    model_load_start = timer()
    ds = Model(models, N_FEATURES, N_CONTEXT, alphabet, BEAM_WIDTH)
--- a/examples/vad_transcriber/wavTranscription.md
+++ b/examples/vad_transcriber/wavTranscription.md
@ -26,7 +26,7 @@ Set the aggressiveness mode, to an integer between 0 and 3.

 ```
 (venv) ~/Deepspeech/examples/vad_transcriber
-$ python3 audioTranscript_cmd.py --aggressive 1 --audio ./audio/guido-van-rossum.wav --model ./models/0.2.0/
+$ python3 audioTranscript_cmd.py --aggressive 1 --audio ./audio/guido-van-rossum.wav --model ./models/0.3.0/


 Filename                       Duration(s)          Inference Time(s)    Model Load Time(s)   LM Load Time(s)
@ -58,4 +58,4 @@ In such a scenario, the GUI tool will not work. The following steps is known to
 (venv) ~/Deepspeech/examples/vad_transcriber$ export PYTHONPATH=/usr/lib/python3/dist-packages/
 (venv) ~/Deepspeech/examples/vad_transcriber$ python3 audioTranscript_gui.py

-```
+```
--- a/native_client/README.md
+++ b/native_client/README.md
@ -133,4 +133,4 @@ make package
 make npm-pack
 ```

-This will create the package `deepspeech-0.2.0.tgz` in `native_client/javascript`.
+This will create the package `deepspeech-0.3.0.tgz` in `native_client/javascript`.
--- a/native_client/client.cc
+++ b/native_client/client.cc
@ -23,7 +23,7 @@
 #define N_CONTEXT 9
 #define BEAM_WIDTH 500
 #define LM_WEIGHT 1.50f
-#define VALID_WORD_COUNT_WEIGHT 2.25f
+#define VALID_WORD_COUNT_WEIGHT 2.10f

 typedef struct {
  const char* string;
--- a/native_client/definitions.mk
+++ b/native_client/definitions.mk
@ -65,7 +65,8 @@ LDFLAGS_NEEDED := -Wl,--no-as-needed
 LDFLAGS_RPATH  := -Wl,-rpath,\$$ORIGIN
 endif
 ifeq ($(OS),Darwin)
-LDFLAGS_NEEDED :=
+CXXFLAGS       += -stdlib=libc++ -mmacosx-version-min=10.10
+LDFLAGS_NEEDED := -stdlib=libc++ -mmacosx-version-min=10.10
 LDFLAGS_RPATH  := -Wl,-rpath,@executable_path
 endif

--- a/native_client/javascript/binding.gyp
+++ b/native_client/javascript/binding.gyp
@ -8,6 +8,21 @@
      ],
      "include_dirs": [
        "../"
+      ],
+      "conditions": [
+        [ "OS=='mac'", {
+            "xcode_settings": {
+              "OTHER_CXXFLAGS": [
+                "-stdlib=libc++",
+                "-mmacosx-version-min=10.10"
+              ],
+              "OTHER_LDFLAGS": [
+                "-stdlib=libc++",
+                "-mmacosx-version-min=10.10"
+              ]
+            }
+          }
+        ]
      ]
    },
    {
--- a/native_client/javascript/client.js
+++ b/native_client/javascript/client.js
@ -19,7 +19,7 @@ const LM_WEIGHT = 1.50;

 // Valid word insertion weight. This is used to lessen the word insertion penalty
 // when the inserted word is part of the vocabulary
-const VALID_WORD_COUNT_WEIGHT = 2.25;
+const VALID_WORD_COUNT_WEIGHT = 2.10;


 // These constants are tied to the shape of the graph used (changing them changes
--- a/native_client/python/client.py
+++ b/native_client/python/client.py
@ -27,7 +27,7 @@ LM_WEIGHT = 1.50

 # Valid word insertion weight. This is used to lessen the word insertion penalty
 # when the inserted word is part of the vocabulary
-VALID_WORD_COUNT_WEIGHT = 2.25
+VALID_WORD_COUNT_WEIGHT = 2.10


 # These constants are tied to the shape of the graph used (changing them changes
--- a/taskcluster/darwin-amd64-cpu-opt.yml
+++ b/taskcluster/darwin-amd64-cpu-opt.yml
@ -6,8 +6,8 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
    - "notify.irc-channel.${notifications.irc}.on-exception"
    - "notify.irc-channel.${notifications.irc}.on-failed"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.osx/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.osx/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.osx/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.osx/artifacts/public/summarize_graph"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/darwin-opt-base.tyml
+++ b/taskcluster/darwin-opt-base.tyml
@ -87,6 +87,7 @@ payload:
        export LC_ALL=C &&
        export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" &&
        export MACOSX_DEPLOYMENT_TARGET=10.10 &&
+        export HOMEBREW_NO_AUTO_UPDATE=1 &&
        env &&
        (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
        git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
@ -97,9 +98,10 @@ payload:
        ${swig.patch_nodejs.osx} &&
        $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
        $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ;
+        export TASKCLUSTER_TASK_EXIT_CODE=$? &&
        mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
        mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
-        cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/
+        cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE

  artifacts:
    - type: "directory"
--- a/taskcluster/docker-build-base.tyml
+++ b/taskcluster/docker-build-base.tyml
@ -43,8 +43,7 @@ then:
          add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" &&
          apt-get -qq update && apt-get -qq -y install docker-ce && mkdir -p /opt/deepspeech &&
          git clone --quiet ${event.head.repo.url} /opt/deepspeech && cd /opt/deepspeech && git checkout --quiet ${event.head.sha} &&
-          curl -fsSL "https://raw.githubusercontent.com/${event.head.user.login}/${event.head.repo.name}/${event.head.sha}/${dockerfile}" > ${dockerfile} &&
-          docker build .
+          docker build --file ${dockerfile} .

    artifacts:
      "public":
--- a/taskcluster/linux-amd64-cpu-opt.yml
+++ b/taskcluster/linux-amd64-cpu-opt.yml
@ -14,8 +14,8 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph"
  scripts:
    build: "taskcluster/host-build.sh"
    package: "taskcluster/package.sh"
--- a/taskcluster/linux-amd64-ctc-opt.yml
+++ b/taskcluster/linux-amd64-ctc-opt.yml
@ -4,8 +4,8 @@ build:
    - "pull_request.synchronize"
    - "pull_request.reopened"
  template_file: linux-opt-base.tyml
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph"
  scripts:
    build: 'taskcluster/decoder-build.sh'
    package: 'taskcluster/decoder-package.sh'
--- a/taskcluster/linux-amd64-cuda-opt.yml
+++ b/taskcluster/linux-amd64-cuda-opt.yml
@ -12,8 +12,8 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cuda/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cuda/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cuda/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cuda/artifacts/public/summarize_graph"
  maxRunTime: 14400
  scripts:
    build: "taskcluster/cuda-build.sh"
--- a/taskcluster/linux-arm64-cpu-opt.yml
+++ b/taskcluster/linux-arm64-cpu-opt.yml
@ -4,8 +4,8 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
    - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.arm64/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.arm64/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/linux-rpi3-cpu-opt.yml
+++ b/taskcluster/linux-rpi3-cpu-opt.yml
@ -4,8 +4,8 @@ build:
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
    - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
    - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.arm/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.arm/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph"
  ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
  system_setup:
    >
--- a/taskcluster/node-package.yml
+++ b/taskcluster/node-package.yml
@ -16,8 +16,8 @@ build:
  system_config:
    >
      ${swig.patch_nodejs.linux}
-  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/home.tar.xz"
-  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/summarize_graph"
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/summarize_graph"
  scripts:
    build: "taskcluster/node-build.sh"
    package: "taskcluster/node-package.sh"
--- a/taskcluster/test-armbian-opt-base.tyml
+++ b/taskcluster/test-armbian-opt-base.tyml
@ -44,7 +44,7 @@ then:
        PIP_DEFAULT_TIMEOUT: "60"
        PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/"
        EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Debian Stretch
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736"

    command:
      - "/bin/bash"
--- a/taskcluster/test-darwin-opt-base.tyml
+++ b/taskcluster/test-darwin-opt-base.tyml
@ -41,7 +41,7 @@ then:
        DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736"

    command:
      - - "/bin/bash"
@ -59,6 +59,7 @@ then:
            export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
            export LC_ALL=C &&
            export MACOSX_DEPLOYMENT_TARGET=10.10 &&
+            export HOMEBREW_NO_AUTO_UPDATE=1 &&
            export PIP_DEFAULT_TIMEOUT=60 &&
            env &&
            git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
@ -66,9 +67,10 @@ then:
            cd $TASKCLUSTER_TASK_DIR &&
            source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-brew-tests.sh && ${extraSystemSetup} &&
            /bin/bash ${build.args.tests_cmdline} ;
+            export TASKCLUSTER_TASK_EXIT_CODE=$? &&
            mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
            mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
-            cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/
+            cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE

    mounts:
      - cacheName: deepspeech-homebrew-bin
--- a/taskcluster/test-linux-opt-base.tyml
+++ b/taskcluster/test-linux-opt-base.tyml
@ -44,7 +44,7 @@ then:
        DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pb
        DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.2.0-prod/output_graph.pbmm
        PIP_DEFAULT_TIMEOUT: "60"
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736"

    command:
      - "/bin/bash"
--- a/taskcluster/test-raspbian-opt-base.tyml
+++ b/taskcluster/test-raspbian-opt-base.tyml
@ -44,7 +44,7 @@ then:
        PIP_DEFAULT_TIMEOUT: "60"
        PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
        EXTRA_PYTHON_CONFIGURE_OPTS: "--with-fpectl" # Required by Raspbian Stretch / PiWheels
-        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5464-geb72bde"
+        EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v1.9.0-rc2-5466-g1b14736"

    command:
      - "/bin/bash"
--- a/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
+++ b/taskcluster/test-training_upstream-linux-amd64-py27mu-opt.yml
@ -7,7 +7,7 @@ build:
      apt-get -qq -y install ${python.packages_trusty.apt}
  args:
    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.14:mu"
-  convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.eb72bde18fd7abdba07c1476ef6f41b78929403d.cpu/artifacts/public/convert_graphdef_memmapped_format"
+  convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.1b14736dd41f2a54339e59793679546981d9ae2c.cpu/artifacts/public/convert_graphdef_memmapped_format"
  metadata:
    name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7 mu"
    description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7 mu, CPU only, optimized version"
--- a/taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml
+++ b/taskcluster/test-training_upstream_frozen-linux-amd64-py36m-opt.yml
@ -1,13 +0,0 @@
-build:
-  template_file: test-linux-opt-base.tyml
-  dependencies:
-    - "linux-amd64-ctc-opt"
-    - "test-training_upstream-linux-amd64-py27mu-opt"
-  system_setup:
-    >
-      apt-get -qq -y install ${python.packages_trusty.apt}
-  args:
-    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 3.6.4:m frozen"
-  metadata:
-    name: "DeepSpeech Linux AMD64 CPU upstream training frozen Py3.6"
-    description: "Training a DeepSpeech LDC93S1 model from frozen model file for Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version"
--- a/tc-brew-tests.sh
+++ b/tc-brew-tests.sh
@ -37,7 +37,7 @@ install_local_homebrew()
    mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}"
    mkdir -p "${HOMEBREW_CACHE}"

-    curl -L https://github.com/Homebrew/brew/tarball/1.7.7 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}"
+    curl -L https://github.com/Homebrew/brew/tarball/1.8.0 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}"
    export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH

    if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then
--- a/tc-tests-utils.sh
+++ b/tc-tests-utils.sh
@ -135,26 +135,30 @@ assert_shows_something()

 assert_correct_ldc93s1()
 {
-  # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS
-  # assert_correct_inference "$1" "she had your dark suit in greasy wash water all year"
-  assert_working_inference "$1" "she had"
+  assert_correct_inference "$1" "she had your dark suit in greasy wash water all year"
+}
+
+assert_correct_ldc93s1_lm()
+{
+  assert_correct_inference "$1" "she had your dark suit in greasy wash water all year"
 }

 assert_correct_multi_ldc93s1()
 {
-  # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS
-  #assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%"
-  #assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%"
-  return 0
+  assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%"
+  assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%"
  ## 8k will output garbage anyway ...
  # assert_shows_something "$1" "/LDC93S1_pcms16le_1_8000.wav%she hayorasryrtl lyreasy asr watal w water all year%"
 }

 assert_correct_ldc93s1_prodmodel()
 {
-  # FIXME: RE-ENABLE AFTER FIXING LM HYPERPARAMETERS
-  #assert_correct_inference "$1" "she had tired or so and greasy wash war or year"
-  assert_working_inference "$1" "she had"
+  assert_correct_inference "$1" "she hadered or so and greasy wash war or year"
+}
+
+assert_correct_ldc93s1_prodmodel_stereo_44k()
+{
+  assert_correct_inference "$1" "she had tered or so and greasy wash war or year"
 }

 assert_correct_warning_upsampling()
@ -185,13 +189,13 @@ run_all_inference_tests()
  assert_correct_ldc93s1 "${phrase_pbmodel_nolm}"

  phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
-  assert_correct_ldc93s1 "${phrase_pbmodel_withlm}"
+  assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}"

  phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
  assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}"

  phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
-  assert_correct_ldc93s1 "${phrase_pbmodel_withlm_stereo_44k}"
+  assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}"

  phrase_pbmodel_nolm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
  assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}"
@ -209,7 +213,7 @@ run_prod_inference_tests()
  assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}"

  phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
-  assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm_stereo_44k}"
+  assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}"

  phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
  assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
@ -261,11 +265,6 @@ download_data()
  cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/smoke_test/vocab.trie ${TASKCLUSTER_TMP_DIR}/trie
 }

-download_for_frozen()
-{
-  wget -O "${TASKCLUSTER_TMP_DIR}/frozen_model.pb" "${DEEPSPEECH_TEST_MODEL}"
-}
-
 download_material()
 {
  target_dir=$1
--- a/tc-train-tests.sh
+++ b/tc-train-tests.sh
@ -6,7 +6,6 @@ source $(dirname "$0")/tc-tests-utils.sh

 pyver_full=$1
 ds=$2
-frozen=$2

 if [ -z "${pyver_full}" ]; then
    echo "No python version given, aborting."
@ -62,17 +61,9 @@ else
 fi;

 pushd ${HOME}/DeepSpeech/ds/
-    if [ "${frozen}" = "frozen" ]; then
-        download_for_frozen
-        time ./bin/run-tc-ldc93s1_frozen.sh
-    else
-        time ./bin/run-tc-ldc93s1_new.sh
-    fi;
+    time ./bin/run-tc-ldc93s1_new.sh 105
 popd

-deactivate
-pyenv uninstall --force ${PYENV_NAME}
-
 cp /tmp/train/output_graph.pb ${TASKCLUSTER_ARTIFACTS}

 if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then
@ -82,3 +73,10 @@ if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then
  /tmp/${convert_graphdef} --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
  cp /tmp/train/output_graph.pbmm ${TASKCLUSTER_ARTIFACTS}
 fi;
+
+pushd ${HOME}/DeepSpeech/ds/
+    time ./bin/run-tc-ldc93s1_checkpoint.sh 105
+popd
+
+deactivate
+pyenv uninstall --force ${PYENV_NAME}