From f4cdd988dfccc5004858c30fab84b0f7c39ad0e6 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 15 Oct 2019 12:38:57 +0200 Subject: [PATCH 1/6] UTF-8 target --- data/lm/trie | 2 +- data/smoke_test/vocab.trie | Bin 172912 -> 172912 bytes .../ctcdecode/ctc_beam_search_decoder.cpp | 21 ++--- native_client/ctcdecode/decoder_utils.cpp | 40 +++++---- native_client/ctcdecode/decoder_utils.h | 15 +++- native_client/ctcdecode/path_trie.cpp | 59 +++++++++---- native_client/ctcdecode/path_trie.h | 17 ++-- native_client/ctcdecode/scorer.cpp | 78 ++++++++++-------- native_client/ctcdecode/scorer.h | 9 +- util/config.py | 7 +- util/evaluate_tools.py | 8 +- util/flags.py | 1 + util/text.py | 45 ++++++++++ 13 files changed, 206 insertions(+), 96 deletions(-) diff --git a/data/lm/trie b/data/lm/trie index 87dd3a32..342c6a9e 100644 --- a/data/lm/trie +++ b/data/lm/trie @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1991108e83cf86830cad118ac9e061aadfc89b73a909a66fe1755f846def556e +oid sha256:f2024f2e83b252df33b5d6c5cced8186abd2764ce7124970c15c0174034c3f2e size 24480560 diff --git a/data/smoke_test/vocab.trie b/data/smoke_test/vocab.trie index 7237e739c3589c810c4ceefedb64eafcf4907831..069a7f8f3eb4a527e24fb917eee6a32e3461a295 100644 GIT binary patch delta 22 dcmexxjO)WOE>>61pb*wZ)>c-=t*lJhiveYD2bur? delta 22 dcmexxjO)WOE>>61pb(Zu)>c-=t*lJhiveY22bll> diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 3c481002..76e394f0 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -36,7 +36,7 @@ DecoderState::init(const Alphabet& alphabet, prefix_root_.reset(root); prefixes_.push_back(root); - if (ext_scorer != nullptr && !ext_scorer->is_character_based()) { + if (ext_scorer != nullptr) { // no need for std::make_shared<>() since Copy() does 'new' behind the doors auto dict_ptr = std::shared_ptr(ext_scorer->dictionary->Copy(true)); root->set_dictionary(dict_ptr); @@ -110,19 +110,10 @@ DecoderState::next(const double *probs, } // language model scoring - if (ext_scorer_ != nullptr && - (c == space_id_ || ext_scorer_->is_character_based())) { - PathTrie *prefix_to_score = nullptr; - // skip scoring the space - if (ext_scorer_->is_character_based()) { - prefix_to_score = prefix_new; - } else { - prefix_to_score = prefix; - } - + if (prefix->character != -1 && ext_scorer_ != nullptr && ext_scorer_->is_scoring_boundary(c)) { float score = 0.0; std::vector ngram; - ngram = ext_scorer_->make_ngram(prefix_to_score); + ngram = ext_scorer_->make_ngram(prefix); bool bos = ngram.size() < ext_scorer_->get_max_order(); score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; log_p += score; @@ -165,10 +156,12 @@ DecoderState::decode() const } // score the last word of each prefix that doesn't end with space - if (ext_scorer_ != nullptr && !ext_scorer_->is_character_based()) { + if (ext_scorer_ != nullptr) { for (size_t i = 0; i < beam_size_ && i < prefixes_copy.size(); ++i) { auto prefix = prefixes_copy[i]; - if (!prefix->is_empty() && prefix->character != space_id_) { + if (prefix->is_empty()) { + scores[prefix] = OOV_SCORE; + } else if (!ext_scorer_->is_scoring_boundary(prefix->parent, prefix->character)) { float score = 0.0; std::vector ngram = ext_scorer_->make_ngram(prefix); bool bos = ngram.size() < ext_scorer_->get_max_order(); diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index 445bfdb2..8a848ac9 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -61,13 +61,18 @@ size_t get_utf8_str_len(const std::string &str) { return str_len; } -std::vector split_utf8_str(const std::string &str) { +// Return weather a byte is a code point boundary (not a continuation byte). +bool byte_is_codepoint_boundary(unsigned char c) { + // only continuation bytes have their most significant bits set to 10 + return (c & 0xC0) != 0x80; +} + +std::vector split_into_codepoints(const std::string &str) { std::vector result; std::string out_str; for (char c : str) { - if ((c & 0xc0) != 0x80) // new UTF-8 character - { + if (byte_is_codepoint_boundary(c)) { if (!out_str.empty()) { result.push_back(out_str); out_str.clear(); @@ -80,6 +85,17 @@ std::vector split_utf8_str(const std::string &str) { return result; } +std::vector split_into_bytes(const std::string &str) { + std::vector result; + + for (char c : str) { + std::string ch(1, c); + result.push_back(ch); + } + + return result; +} + std::vector split_str(const std::string &s, const std::string &delim) { std::vector result; @@ -144,27 +160,23 @@ void add_word_to_fst(const std::vector &word, bool add_word_to_dictionary( const std::string &word, const std::unordered_map &char_map, - bool add_space, + bool utf8, int SPACE_ID, fst::StdVectorFst *dictionary) { - auto characters = split_utf8_str(word); + auto characters = utf8 ? split_into_bytes(word) : split_into_codepoints(word); std::vector int_word; for (auto &c : characters) { - if (c == " ") { - int_word.push_back(SPACE_ID); + auto int_c = char_map.find(c); + if (int_c != char_map.end()) { + int_word.push_back(int_c->second); } else { - auto int_c = char_map.find(c); - if (int_c != char_map.end()) { - int_word.push_back(int_c->second); - } else { - return false; // return without adding - } + return false; // return without adding } } - if (add_space) { + if (!utf8) { int_word.push_back(SPACE_ID); } diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index 797810b7..ebcf2dcf 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -81,20 +81,27 @@ size_t get_utf8_str_len(const std::string &str); std::vector split_str(const std::string &s, const std::string &delim); -/* Splits string into vector of strings representing - * UTF-8 characters (not same as chars) +/* Splits string into vector of UTF-8 byte sequences representing a single + * Unicode codepoint. */ -std::vector split_utf8_str(const std::string &str); +std::vector split_into_codepoints(const std::string &str); + +/* Splits string into bytes. + */ +std::vector split_into_bytes(const std::string &str); // Add a word in index to the dicionary of fst void add_word_to_fst(const std::vector &word, fst::StdVectorFst *dictionary); +// Return weather a byte is a code point boundary (not a continuation byte). +bool byte_is_codepoint_boundary(unsigned char c); + // Add a word in string to dictionary bool add_word_to_dictionary( const std::string &word, const std::unordered_map &char_map, - bool add_space, + bool utf8, int SPACE_ID, fst::StdVectorFst *dictionary); #endif // DECODER_UTILS_H diff --git a/native_client/ctcdecode/path_trie.cpp b/native_client/ctcdecode/path_trie.cpp index c1ad2441..888ab34d 100644 --- a/native_client/ctcdecode/path_trie.cpp +++ b/native_client/ctcdecode/path_trie.cpp @@ -111,25 +111,54 @@ PathTrie* PathTrie::get_path_trie(int new_char, int new_timestep, float cur_log_ } } -PathTrie* PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { - return get_path_vec(output, timesteps, ROOT_); -} - -PathTrie* PathTrie::get_path_vec(std::vector& output, - std::vector& timesteps, - int stop, - size_t max_steps) { - if (character == stop || character == ROOT_ || output.size() == max_steps) { - std::reverse(output.begin(), output.end()); - std::reverse(timesteps.begin(), timesteps.end()); - return this; - } else { +void PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + if (parent != nullptr) { + parent->get_path_vec(output, timesteps); + } + if (character != ROOT_) { output.push_back(character); timesteps.push_back(timestep); - return parent->get_path_vec(output, timesteps, stop, max_steps); } } +PathTrie* PathTrie::get_prev_grapheme(std::vector& output, + std::vector& timesteps) +{ + PathTrie* stop = this; + if (character == ROOT_) { + return stop; + } + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + //FIXME: use Alphabet instead of hardcoding +1 here + if (!byte_is_codepoint_boundary(character+1)) { + stop = parent->get_prev_grapheme(output, timesteps); + } + output.push_back(character); + timesteps.push_back(timestep); + return stop; +} + +PathTrie* PathTrie::get_prev_word(std::vector& output, + std::vector& timesteps, + int space_id) +{ + PathTrie* stop = this; + if (character == space_id || character == ROOT_) { + return stop; + } + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + if (parent != nullptr) { + stop = parent->get_prev_word(output, timesteps, space_id); + } + output.push_back(character); + timesteps.push_back(timestep); + return stop; +} + void PathTrie::iterate_to_vec(std::vector& output) { if (exists_) { log_prob_b_prev = log_prob_b_cur; @@ -189,7 +218,7 @@ void PathTrie::print(const Alphabet& a) { std::string tr; printf("characters:\t "); for (PathTrie* el : chain) { - printf("%X ", el->character); + printf("%X ", (unsigned char)(el->character)); if (el->character != ROOT_) { tr.append(a.StringFromLabel(el->character)); } diff --git a/native_client/ctcdecode/path_trie.h b/native_client/ctcdecode/path_trie.h index 04cbfdd5..826f652b 100644 --- a/native_client/ctcdecode/path_trie.h +++ b/native_client/ctcdecode/path_trie.h @@ -26,14 +26,17 @@ public: // get new prefix after appending new char PathTrie* get_path_trie(int new_char, int new_timestep, float log_prob_c, bool reset = true); - // get the prefix in index from root to current node - PathTrie* get_path_vec(std::vector& output, std::vector& timesteps); + // get the prefix data in correct time order from root to current node + void get_path_vec(std::vector& output, std::vector& timesteps); - // get the prefix in index from some stop node to current nodel - PathTrie* get_path_vec(std::vector& output, - std::vector& timesteps, - int stop, - size_t max_steps = std::numeric_limits::max()); + // get the prefix data in correct time order from beginning of last grapheme to current node + PathTrie* get_prev_grapheme(std::vector& output, + std::vector& timesteps); + + // get the prefix data in correct time order from beginning of last word to current node + PathTrie* get_prev_word(std::vector& output, + std::vector& timesteps, + int space_id); // update log probs void iterate_to_vec(std::vector& output); diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index c265b430..65275142 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -27,7 +27,7 @@ using namespace lm::ngram; static const int32_t MAGIC = 'TRIE'; -static const int32_t FILE_VERSION = 4; +static const int32_t FILE_VERSION = 5; int Scorer::init(double alpha, @@ -86,16 +86,21 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path) language_model_.reset(lm::ngram::LoadVirtual(filename, config)); auto vocab = enumerate.vocabulary; for (size_t i = 0; i < vocab.size(); ++i) { - if (is_character_based_ && vocab[i] != UNK_TOKEN && - vocab[i] != START_TOKEN && vocab[i] != END_TOKEN && - get_utf8_str_len(enumerate.vocabulary[i]) > 1) { - is_character_based_ = false; + if (vocab[i] != UNK_TOKEN && + vocab[i] != START_TOKEN && + vocab[i] != END_TOKEN && + get_utf8_str_len(vocab[i]) > 1) { + is_utf8_mode_ = false; + break; } } - // fill the dictionary for FST - if (!is_character_based()) { - fill_dictionary(vocab, true); + + if (alphabet_.GetSize() != 255) { + is_utf8_mode_ = false; } + + // Add spaces only in word-based scoring + fill_dictionary(vocab); } else { config.load_method = util::LoadMethod::LAZY; language_model_.reset(lm::ngram::LoadVirtual(filename, config)); @@ -121,14 +126,12 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path) throw 1; } - fin.read(reinterpret_cast(&is_character_based_), sizeof(is_character_based_)); + fin.read(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); - if (!is_character_based_) { - fst::FstReadOptions opt; - opt.mode = fst::FstReadOptions::MAP; - opt.source = trie_path; - dictionary.reset(FstType::Read(fin, opt)); - } + fst::FstReadOptions opt; + opt.mode = fst::FstReadOptions::MAP; + opt.source = trie_path; + dictionary.reset(FstType::Read(fin, opt)); } max_order_ = language_model_->Order(); @@ -139,12 +142,20 @@ void Scorer::save_dictionary(const std::string& path) std::ofstream fout(path, std::ios::binary); fout.write(reinterpret_cast(&MAGIC), sizeof(MAGIC)); fout.write(reinterpret_cast(&FILE_VERSION), sizeof(FILE_VERSION)); - fout.write(reinterpret_cast(&is_character_based_), sizeof(is_character_based_)); - if (!is_character_based_) { - fst::FstWriteOptions opt; - opt.align = true; - opt.source = path; - dictionary->Write(fout, opt); + fout.write(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); + fst::FstWriteOptions opt; + opt.align = true; + opt.source = path; + dictionary->Write(fout, opt); +} + +bool Scorer::is_scoring_boundary(size_t label) +{ + if (is_utf8_mode()) { + unsigned char byte_val = alphabet_.StringFromLabel(label)[0]; + return byte_is_codepoint_boundary(byte_val); + } else { + return label == SPACE_ID_; } } @@ -251,8 +262,8 @@ std::vector Scorer::split_labels(const std::vector& labels) std::string s = alphabet_.LabelsToString(labels); std::vector words; - if (is_character_based_) { - words = split_utf8_str(s); + if (is_utf8_mode_) { + words = split_into_bytes(s); } else { words = split_str(s, " "); } @@ -266,30 +277,29 @@ std::vector Scorer::make_ngram(PathTrie* prefix) PathTrie* new_node = nullptr; for (int order = 0; order < max_order_; order++) { + if (!current_node || current_node->character == -1) { + break; + } + std::vector prefix_vec; std::vector prefix_steps; - if (is_character_based_) { - new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_, 1); - current_node = new_node; + if (is_utf8_mode_) { + new_node = current_node->get_prev_grapheme(prefix_vec, prefix_steps); } else { - new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_); - current_node = new_node->parent; // Skipping spaces + new_node = current_node->get_prev_word(prefix_vec, prefix_steps, SPACE_ID_); } + current_node = new_node->parent; // reconstruct word std::string word = alphabet_.LabelsToString(prefix_vec); ngram.push_back(word); - - if (new_node->character == -1) { - break; - } } std::reverse(ngram.begin(), ngram.end()); return ngram; } -void Scorer::fill_dictionary(const std::vector& vocabulary, bool add_space) +void Scorer::fill_dictionary(const std::vector& vocabulary) { // ConstFst is immutable, so we need to use a MutableFst to create the trie, // and then we convert to a ConstFst for the decoder and for storing on disk. @@ -297,7 +307,7 @@ void Scorer::fill_dictionary(const std::vector& vocabulary, bool ad // For each unigram convert to ints and put in trie for (const auto& word : vocabulary) { if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) { - add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary); + add_word_to_dictionary(word, char_map_, is_utf8_mode_, SPACE_ID_ + 1, &dictionary); } } diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index 5540138c..9b74056d 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -77,7 +77,7 @@ public: size_t get_max_order() const { return max_order_; } // retrun true if the language model is character based - bool is_character_based() const { return is_character_based_; } + bool is_utf8_mode() const { return is_utf8_mode_; } // reset params alpha & beta void reset_params(float alpha, float beta); @@ -92,6 +92,9 @@ public: // save dictionary in file void save_dictionary(const std::string &path); + // return weather this label represents a boundary where beam scoring should happen + bool is_scoring_boundary(size_t label); + // language model weight double alpha = 0.; // word insertion weight @@ -108,11 +111,11 @@ protected: void load_lm(const std::string &lm_path); // fill dictionary for FST - void fill_dictionary(const std::vector &vocabulary, bool add_space); + void fill_dictionary(const std::vector &vocabulary); private: std::unique_ptr language_model_; - bool is_character_based_ = true; + bool is_utf8_mode_ = true; size_t max_order_ = 0; int SPACE_ID_; diff --git a/util/config.py b/util/config.py index fe58e31b..eab52ecb 100755 --- a/util/config.py +++ b/util/config.py @@ -10,7 +10,7 @@ from xdg import BaseDirectory as xdg from util.flags import FLAGS from util.gpu import get_available_gpus from util.logging import log_error -from util.text import Alphabet +from util.text import Alphabet, UTF8Alphabet class ConfigSingleton: _config = None @@ -63,7 +63,10 @@ def initialize_globals(): if not c.available_devices: c.available_devices = [c.cpu_device] - c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path)) + if FLAGS.utf8: + c.alphabet = UTF8Alphabet() + else: + c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path)) # Geometric Constants # =================== diff --git a/util/evaluate_tools.py b/util/evaluate_tools.py index ed69578a..7f6a8ffb 100644 --- a/util/evaluate_tools.py +++ b/util/evaluate_tools.py @@ -6,6 +6,7 @@ from multiprocessing.dummy import Pool from attrdict import AttrDict +from util.flags import FLAGS from util.text import levenshtein @@ -67,7 +68,10 @@ def calculate_report(wav_filenames, labels, decodings, losses): # Order the remaining items by their loss (lowest loss on top) samples.sort(key=lambda s: s.loss) - # Then order by WER (highest WER on top) - samples.sort(key=lambda s: s.wer, reverse=True) + # Then order by descending WER/CER + if FLAGS.utf8: + samples.sort(key=lambda s: s.cer, reverse=True) + else: + samples.sort(key=lambda s: s.wer, reverse=True) return samples_wer, samples_cer, samples diff --git a/util/flags.py b/util/flags.py index 6a8caf3b..a2d2cf2a 100644 --- a/util/flags.py +++ b/util/flags.py @@ -132,6 +132,7 @@ def create_flags(): # Decoder + f.DEFINE_boolean('utf8', False, 'enable UTF-8 mode. When this is used the model outputs UTF-8 sequences directly rather than using an alphabet mapping.') f.DEFINE_string('alphabet_config_path', 'data/alphabet.txt', 'path to the configuration file specifying the alphabet used by the network. See the comment in data/alphabet.txt for a description of the format.') f.DEFINE_string('lm_binary_path', 'data/lm/lm.binary', 'path to the language model binary file created with KenLM') f.DEFINE_alias('lm', 'lm_binary_path') diff --git a/util/text.py b/util/text.py index 7928a990..d3be7eb8 100644 --- a/util/text.py +++ b/util/text.py @@ -76,6 +76,51 @@ class Alphabet(object): return self._config_file +class UTF8Alphabet(object): + @staticmethod + def _string_from_label(_): + assert False + + @staticmethod + def _label_from_string(_): + assert False + + @staticmethod + def encode(string): + # 0 never happens in the data, so we can shift values by one, use 255 for + # the CTC blank, and keep the alphabet size = 256 + return np.frombuffer(string.encode('utf-8'), np.uint8).astype(np.int32) - 1 + + @staticmethod + def decode(labels): + # And here we need to shift back up + return bytes(np.asarray(labels, np.uint8) + 1).decode('utf-8', errors='replace') + + @staticmethod + def size(): + return 255 + + @staticmethod + def serialize(): + res = bytearray() + res += struct.pack(' Date: Fri, 8 Nov 2019 16:17:28 +0100 Subject: [PATCH 2/6] Score prefix as soon as a grapheme is formed rather than 1 byte later --- .../ctcdecode/ctc_beam_search_decoder.cpp | 16 ++++++++++-- native_client/ctcdecode/path_trie.cpp | 16 +++++++++++- native_client/ctcdecode/path_trie.h | 3 +++ native_client/ctcdecode/scorer.cpp | 25 ++++++++++++++++--- native_client/ctcdecode/scorer.h | 4 +-- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 76e394f0..41faa565 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -109,11 +109,23 @@ DecoderState::next(const double *probs, log_p = log_prob_c + prefix->score; } + // skip scoring the space in word based LMs + PathTrie* prefix_to_score; + if (ext_scorer_->is_utf8_mode()) { + prefix_to_score = prefix_new; + } else { + prefix_to_score = prefix; + } + + // check if we need to score + bool is_scoring_boundary = ext_scorer_ != nullptr && + ext_scorer_->is_scoring_boundary(prefix_to_score, c); + // language model scoring - if (prefix->character != -1 && ext_scorer_ != nullptr && ext_scorer_->is_scoring_boundary(c)) { + if (is_scoring_boundary) { float score = 0.0; std::vector ngram; - ngram = ext_scorer_->make_ngram(prefix); + ngram = ext_scorer_->make_ngram(prefix_to_score); bool bos = ngram.size() < ext_scorer_->get_max_order(); score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; log_p += score; diff --git a/native_client/ctcdecode/path_trie.cpp b/native_client/ctcdecode/path_trie.cpp index 888ab34d..749b5a31 100644 --- a/native_client/ctcdecode/path_trie.cpp +++ b/native_client/ctcdecode/path_trie.cpp @@ -133,7 +133,7 @@ PathTrie* PathTrie::get_prev_grapheme(std::vector& output, // Recursive call: recurse back until stop condition, then append data in // correct order as we walk back down the stack in the lines below. //FIXME: use Alphabet instead of hardcoding +1 here - if (!byte_is_codepoint_boundary(character+1)) { + if (!byte_is_codepoint_boundary(character + 1)) { stop = parent->get_prev_grapheme(output, timesteps); } output.push_back(character); @@ -141,6 +141,20 @@ PathTrie* PathTrie::get_prev_grapheme(std::vector& output, return stop; } +int PathTrie::distance_to_codepoint_boundary(unsigned char *first_byte) +{ + //FIXME: use Alphabet instead of hardcoding +1 here + if (byte_is_codepoint_boundary(character + 1)) { + *first_byte = (unsigned char)character + 1; + return 1; + } + if (parent != nullptr && parent->character != ROOT_) { + return 1 + parent->distance_to_codepoint_boundary(first_byte); + } + assert(false); // unreachable + return 0; +} + PathTrie* PathTrie::get_prev_word(std::vector& output, std::vector& timesteps, int space_id) diff --git a/native_client/ctcdecode/path_trie.h b/native_client/ctcdecode/path_trie.h index 826f652b..e087b1a6 100644 --- a/native_client/ctcdecode/path_trie.h +++ b/native_client/ctcdecode/path_trie.h @@ -33,6 +33,9 @@ public: PathTrie* get_prev_grapheme(std::vector& output, std::vector& timesteps); + // get the distance from current node to the first codepoint boundary, and the byte value at the boundary + int distance_to_codepoint_boundary(unsigned char *first_byte); + // get the prefix data in correct time order from beginning of last word to current node PathTrie* get_prev_word(std::vector& output, std::vector& timesteps, diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index 65275142..0538340e 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -149,13 +149,30 @@ void Scorer::save_dictionary(const std::string& path) dictionary->Write(fout, opt); } -bool Scorer::is_scoring_boundary(size_t label) +bool Scorer::is_scoring_boundary(PathTrie* prefix, size_t new_label) { if (is_utf8_mode()) { - unsigned char byte_val = alphabet_.StringFromLabel(label)[0]; - return byte_is_codepoint_boundary(byte_val); + if (prefix->character == -1) { + return false; + } + unsigned char first_byte; + int distance_to_boundary = prefix->distance_to_codepoint_boundary(&first_byte); + int needed_bytes; + if ((first_byte >> 3) == 0x1E) { + needed_bytes = 4; + } else if ((first_byte >> 4) == 0x0E) { + needed_bytes = 3; + } else if ((first_byte >> 5) == 0x06) { + needed_bytes = 2; + } else if ((first_byte >> 7) == 0x00) { + needed_bytes = 1; + } else { + assert(false); // invalid byte sequence. should be unreachable, disallowed by vocabulary/trie + return false; + } + return distance_to_boundary == needed_bytes; } else { - return label == SPACE_ID_; + return new_label == SPACE_ID_; } } diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index 9b74056d..f0483c7c 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -92,8 +92,8 @@ public: // save dictionary in file void save_dictionary(const std::string &path); - // return weather this label represents a boundary where beam scoring should happen - bool is_scoring_boundary(size_t label); + // return weather this step represents a boundary where beam scoring should happen + bool is_scoring_boundary(PathTrie* prefix, size_t new_label); // language model weight double alpha = 0.; From 0e6952c3a8359d6df25b3786f5aedd4a40f282d3 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sat, 9 Nov 2019 14:35:24 +0100 Subject: [PATCH 3/6] Avoid reconstructing strings twice on decode --- .../ctcdecode/ctc_beam_search_decoder.cpp | 27 ++++++++++--------- native_client/ctcdecode/decoder_utils.cpp | 15 ----------- native_client/ctcdecode/decoder_utils.h | 5 ---- native_client/ctcdecode/scorer.cpp | 4 +-- native_client/ctcdecode/scorer.h | 2 +- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 41faa565..31999078 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -192,27 +192,30 @@ DecoderState::decode() const std::bind(prefix_compare_external, _1, _2, scores)); //TODO: expose this as an API parameter - const int top_paths = 1; + const size_t top_paths = 1; + size_t num_returned = std::min(num_prefixes, top_paths); + + std::vector outputs; + outputs.reserve(num_returned); // compute aproximate ctc score as the return score, without affecting the // return order of decoding result. To delete when decoder gets stable. - for (size_t i = 0; i < top_paths && i < prefixes_copy.size(); ++i) { + for (size_t i = 0; i < num_returned; ++i) { + Output output; + prefixes_copy[i]->get_path_vec(output.tokens, output.timesteps); double approx_ctc = scores[prefixes_copy[i]]; if (ext_scorer_ != nullptr) { - std::vector output; - std::vector timesteps; - prefixes_copy[i]->get_path_vec(output, timesteps); - auto prefix_length = output.size(); - auto words = ext_scorer_->split_labels(output); - // remove word insert - approx_ctc = approx_ctc - prefix_length * ext_scorer_->beta; - // remove language model weight: + auto words = ext_scorer_->split_labels_into_scored_units(output.tokens); + // remove term insertion weight + approx_ctc -= words.size() * ext_scorer_->beta; + // remove language model weight approx_ctc -= (ext_scorer_->get_sent_log_prob(words)) * ext_scorer_->alpha; } - prefixes_copy[i]->approx_ctc = approx_ctc; + output.confidence = -approx_ctc; + outputs.push_back(output); } - return get_beam_search_result(prefixes_copy, top_paths); + return outputs; } std::vector ctc_beam_search_decoder( diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index 8a848ac9..be810c07 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -38,21 +38,6 @@ std::vector> get_pruned_log_probs( return log_prob_idx; } - -std::vector get_beam_search_result( - const std::vector &prefixes, - size_t top_paths) { - std::vector output_vecs; - for (size_t i = 0; i < top_paths && i < prefixes.size(); ++i) { - Output output; - prefixes[i]->get_path_vec(output.tokens, output.timesteps); - output.confidence = -prefixes[i]->approx_ctc; - output_vecs.push_back(output); - } - - return output_vecs; -} - size_t get_utf8_str_len(const std::string &str) { size_t str_len = 0; for (char c : str) { diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index ebcf2dcf..ec0a93fc 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -59,11 +59,6 @@ std::vector> get_pruned_log_probs( double cutoff_prob, size_t cutoff_top_n); -// Get beam search result from prefixes in trie tree -std::vector get_beam_search_result( - const std::vector &prefixes, - size_t top_paths); - // Functor for prefix comparsion bool prefix_compare(const PathTrie *x, const PathTrie *y); diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index 0538340e..82197c79 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -273,14 +273,14 @@ void Scorer::reset_params(float alpha, float beta) this->beta = beta; } -std::vector Scorer::split_labels(const std::vector& labels) +std::vector Scorer::split_labels_into_scored_units(const std::vector& labels) { if (labels.empty()) return {}; std::string s = alphabet_.LabelsToString(labels); std::vector words; if (is_utf8_mode_) { - words = split_into_bytes(s); + words = split_into_codepoints(s); } else { words = split_str(s, " "); } diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index f0483c7c..96c092b0 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -87,7 +87,7 @@ public: // trransform the labels in index to the vector of words (word based lm) or // the vector of characters (character based lm) - std::vector split_labels(const std::vector &labels); + std::vector split_labels_into_scored_units(const std::vector &labels); // save dictionary in file void save_dictionary(const std::string &path); From d2eb305b73823efacb3f8de2b480346017c50cd7 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 12 Nov 2019 21:56:42 +0100 Subject: [PATCH 4/6] Address review comment and add missing check for presence of scorer --- .../ctcdecode/ctc_beam_search_decoder.cpp | 38 +++++++++---------- native_client/ctcdecode/decoder_utils.cpp | 6 --- native_client/ctcdecode/decoder_utils.h | 7 +++- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 31999078..5a2c834e 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -109,27 +109,25 @@ DecoderState::next(const double *probs, log_p = log_prob_c + prefix->score; } - // skip scoring the space in word based LMs - PathTrie* prefix_to_score; - if (ext_scorer_->is_utf8_mode()) { - prefix_to_score = prefix_new; - } else { - prefix_to_score = prefix; - } + if (ext_scorer_ != nullptr) { + // skip scoring the space in word based LMs + PathTrie* prefix_to_score; + if (ext_scorer_->is_utf8_mode()) { + prefix_to_score = prefix_new; + } else { + prefix_to_score = prefix; + } - // check if we need to score - bool is_scoring_boundary = ext_scorer_ != nullptr && - ext_scorer_->is_scoring_boundary(prefix_to_score, c); - - // language model scoring - if (is_scoring_boundary) { - float score = 0.0; - std::vector ngram; - ngram = ext_scorer_->make_ngram(prefix_to_score); - bool bos = ngram.size() < ext_scorer_->get_max_order(); - score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; - log_p += score; - log_p += ext_scorer_->beta; + // language model scoring + if (ext_scorer_->is_scoring_boundary(prefix_to_score, c)) { + float score = 0.0; + std::vector ngram; + ngram = ext_scorer_->make_ngram(prefix_to_score); + bool bos = ngram.size() < ext_scorer_->get_max_order(); + score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; + log_p += score; + log_p += ext_scorer_->beta; + } } prefix_new->log_prob_nb_cur = diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index be810c07..ed244c3a 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -46,12 +46,6 @@ size_t get_utf8_str_len(const std::string &str) { return str_len; } -// Return weather a byte is a code point boundary (not a continuation byte). -bool byte_is_codepoint_boundary(unsigned char c) { - // only continuation bytes have their most significant bits set to 10 - return (c & 0xC0) != 0x80; -} - std::vector split_into_codepoints(const std::string &str) { std::vector result; std::string out_str; diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index ec0a93fc..3ba1d7e6 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -89,8 +89,11 @@ std::vector split_into_bytes(const std::string &str); void add_word_to_fst(const std::vector &word, fst::StdVectorFst *dictionary); -// Return weather a byte is a code point boundary (not a continuation byte). -bool byte_is_codepoint_boundary(unsigned char c); +// Return whether a byte is a code point boundary (not a continuation byte). +inline bool byte_is_codepoint_boundary(unsigned char c) { + // only continuation bytes have their most significant bits set to 10 + return (c & 0xC0) != 0x80; +} // Add a word in string to dictionary bool add_word_to_dictionary( From 7cd8e20045412ff95106f63c9cc784f4f2229f94 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 13 Nov 2019 10:37:50 +0100 Subject: [PATCH 6/6] Re-export model used by examples with UTF-8 trie --- taskcluster/examples-base.tyml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/examples-base.tyml b/taskcluster/examples-base.tyml index 4e7c24e1..8e149a9d 100644 --- a/taskcluster/examples-base.tyml +++ b/taskcluster/examples-base.tyml @@ -30,7 +30,7 @@ then: image: ${build.docker_image} env: - DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.11/models.tar.gz" + DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.14/models.tar.gz" DEEPSPEECH_AUDIO: "https://github.com/mozilla/DeepSpeech/releases/download/v0.4.1/audio-0.4.1.tar.gz" PIP_DEFAULT_TIMEOUT: "60"