From 7c0354483ed4eac8cfacfb8e72dbedb56f3082b2 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 15 Jan 2020 23:28:35 +0100 Subject: [PATCH] Stop including vocabulary data in LM.binary. --- data/lm/README.rst | 2 +- data/lm/generate_lm.py | 1 + data/lm/lm.binary | 4 +- native_client/ctcdecode/scorer.cpp | 79 +++++++++++------------------- 4 files changed, 32 insertions(+), 54 deletions(-) diff --git a/data/lm/README.rst b/data/lm/README.rst index add2b195..bd2c2d3b 100644 --- a/data/lm/README.rst +++ b/data/lm/README.rst @@ -1,5 +1,5 @@ -lm.binary was generated from the LibriSpeech normalized LM training text, available `here `_\ , using the `generate_lm.py` script (will generate lm.binary in the folder it is run from). KenLM's built binaries must be in your PATH (lmplz, build_binary, filter). +lm.binary was generated from the LibriSpeech normalized LM training text, available `here `_\ , using the `generate_lm.py` script (will generate lm.binary in the folder it is run from). `KenLM `_'s built binaries must be in your PATH (lmplz, build_binary, filter). The trie was then generated from the vocabulary of the language model: diff --git a/data/lm/generate_lm.py b/data/lm/generate_lm.py index 82fe6468..6dc320a5 100644 --- a/data/lm/generate_lm.py +++ b/data/lm/generate_lm.py @@ -50,6 +50,7 @@ def main(): subprocess.check_call([ 'build_binary', '-a', '255', '-q', '8', + '-v', 'trie', filtered_path, 'lm.binary' diff --git a/data/lm/lm.binary b/data/lm/lm.binary index 16e7d6d9..22584cb1 100644 --- a/data/lm/lm.binary +++ b/data/lm/lm.binary @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a24953ce3f013bbf5f4a1c9f5a0e5482bc56eaa81638276de522f39e62ff3a56 -size 945699324 +oid sha256:cc8d9e5f49e2fa05c56cc928520c6c79cb78ff95226ec9a07785b3a28d1a680b +size 941235601 diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index 82197c79..79dbef8b 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -77,63 +77,40 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path) VALID_CHECK_EQ(access(filename, R_OK), 0, "Invalid language model path"); bool has_trie = trie_path.size() && access(trie_path.c_str(), R_OK) == 0; + VALID_CHECK(has_trie, "Invalid trie path"); lm::ngram::Config config; + config.load_method = util::LoadMethod::LAZY; + language_model_.reset(lm::ngram::LoadVirtual(filename, config)); - if (!has_trie) { // no trie was specified, build it now - RetrieveStrEnumerateVocab enumerate; - config.enumerate_vocab = &enumerate; - language_model_.reset(lm::ngram::LoadVirtual(filename, config)); - auto vocab = enumerate.vocabulary; - for (size_t i = 0; i < vocab.size(); ++i) { - if (vocab[i] != UNK_TOKEN && - vocab[i] != START_TOKEN && - vocab[i] != END_TOKEN && - get_utf8_str_len(vocab[i]) > 1) { - is_utf8_mode_ = false; - break; - } - } + // Read metadata and trie from file + std::ifstream fin(trie_path, std::ios::binary); - if (alphabet_.GetSize() != 255) { - is_utf8_mode_ = false; - } - - // Add spaces only in word-based scoring - fill_dictionary(vocab); - } else { - config.load_method = util::LoadMethod::LAZY; - language_model_.reset(lm::ngram::LoadVirtual(filename, config)); - - // Read metadata and trie from file - std::ifstream fin(trie_path, std::ios::binary); - - int magic; - fin.read(reinterpret_cast(&magic), sizeof(magic)); - if (magic != MAGIC) { - std::cerr << "Error: Can't parse trie file, invalid header. Try updating " - "your trie file." << std::endl; - throw 1; - } - - int version; - fin.read(reinterpret_cast(&version), sizeof(version)); - if (version != FILE_VERSION) { - std::cerr << "Error: Trie file version mismatch (" << version - << " instead of expected " << FILE_VERSION - << "). Update your trie file." - << std::endl; - throw 1; - } - - fin.read(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); - - fst::FstReadOptions opt; - opt.mode = fst::FstReadOptions::MAP; - opt.source = trie_path; - dictionary.reset(FstType::Read(fin, opt)); + int magic; + fin.read(reinterpret_cast(&magic), sizeof(magic)); + if (magic != MAGIC) { + std::cerr << "Error: Can't parse trie file, invalid header. Try updating " + "your trie file." << std::endl; + throw 1; } + int version; + fin.read(reinterpret_cast(&version), sizeof(version)); + if (version != FILE_VERSION) { + std::cerr << "Error: Trie file version mismatch (" << version + << " instead of expected " << FILE_VERSION + << "). Update your trie file." + << std::endl; + throw 1; + } + + fin.read(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); + + fst::FstReadOptions opt; + opt.mode = fst::FstReadOptions::MAP; + opt.source = trie_path; + dictionary_.reset(FstType::Read(fin, opt)); + max_order_ = language_model_->Order(); }