diff --git a/data/lm/trie b/data/lm/trie index 87dd3a32..342c6a9e 100644 --- a/data/lm/trie +++ b/data/lm/trie @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1991108e83cf86830cad118ac9e061aadfc89b73a909a66fe1755f846def556e +oid sha256:f2024f2e83b252df33b5d6c5cced8186abd2764ce7124970c15c0174034c3f2e size 24480560 diff --git a/data/smoke_test/vocab.trie b/data/smoke_test/vocab.trie index 7237e739..069a7f8f 100644 Binary files a/data/smoke_test/vocab.trie and b/data/smoke_test/vocab.trie differ diff --git a/native_client/ctcdecode/ctc_beam_search_decoder.cpp b/native_client/ctcdecode/ctc_beam_search_decoder.cpp index 3c481002..5a2c834e 100644 --- a/native_client/ctcdecode/ctc_beam_search_decoder.cpp +++ b/native_client/ctcdecode/ctc_beam_search_decoder.cpp @@ -36,7 +36,7 @@ DecoderState::init(const Alphabet& alphabet, prefix_root_.reset(root); prefixes_.push_back(root); - if (ext_scorer != nullptr && !ext_scorer->is_character_based()) { + if (ext_scorer != nullptr) { // no need for std::make_shared<>() since Copy() does 'new' behind the doors auto dict_ptr = std::shared_ptr(ext_scorer->dictionary->Copy(true)); root->set_dictionary(dict_ptr); @@ -109,24 +109,25 @@ DecoderState::next(const double *probs, log_p = log_prob_c + prefix->score; } - // language model scoring - if (ext_scorer_ != nullptr && - (c == space_id_ || ext_scorer_->is_character_based())) { - PathTrie *prefix_to_score = nullptr; - // skip scoring the space - if (ext_scorer_->is_character_based()) { + if (ext_scorer_ != nullptr) { + // skip scoring the space in word based LMs + PathTrie* prefix_to_score; + if (ext_scorer_->is_utf8_mode()) { prefix_to_score = prefix_new; } else { prefix_to_score = prefix; } - float score = 0.0; - std::vector ngram; - ngram = ext_scorer_->make_ngram(prefix_to_score); - bool bos = ngram.size() < ext_scorer_->get_max_order(); - score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; - log_p += score; - log_p += ext_scorer_->beta; + // language model scoring + if (ext_scorer_->is_scoring_boundary(prefix_to_score, c)) { + float score = 0.0; + std::vector ngram; + ngram = ext_scorer_->make_ngram(prefix_to_score); + bool bos = ngram.size() < ext_scorer_->get_max_order(); + score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha; + log_p += score; + log_p += ext_scorer_->beta; + } } prefix_new->log_prob_nb_cur = @@ -165,10 +166,12 @@ DecoderState::decode() const } // score the last word of each prefix that doesn't end with space - if (ext_scorer_ != nullptr && !ext_scorer_->is_character_based()) { + if (ext_scorer_ != nullptr) { for (size_t i = 0; i < beam_size_ && i < prefixes_copy.size(); ++i) { auto prefix = prefixes_copy[i]; - if (!prefix->is_empty() && prefix->character != space_id_) { + if (prefix->is_empty()) { + scores[prefix] = OOV_SCORE; + } else if (!ext_scorer_->is_scoring_boundary(prefix->parent, prefix->character)) { float score = 0.0; std::vector ngram = ext_scorer_->make_ngram(prefix); bool bos = ngram.size() < ext_scorer_->get_max_order(); @@ -187,27 +190,30 @@ DecoderState::decode() const std::bind(prefix_compare_external, _1, _2, scores)); //TODO: expose this as an API parameter - const int top_paths = 1; + const size_t top_paths = 1; + size_t num_returned = std::min(num_prefixes, top_paths); + + std::vector outputs; + outputs.reserve(num_returned); // compute aproximate ctc score as the return score, without affecting the // return order of decoding result. To delete when decoder gets stable. - for (size_t i = 0; i < top_paths && i < prefixes_copy.size(); ++i) { + for (size_t i = 0; i < num_returned; ++i) { + Output output; + prefixes_copy[i]->get_path_vec(output.tokens, output.timesteps); double approx_ctc = scores[prefixes_copy[i]]; if (ext_scorer_ != nullptr) { - std::vector output; - std::vector timesteps; - prefixes_copy[i]->get_path_vec(output, timesteps); - auto prefix_length = output.size(); - auto words = ext_scorer_->split_labels(output); - // remove word insert - approx_ctc = approx_ctc - prefix_length * ext_scorer_->beta; - // remove language model weight: + auto words = ext_scorer_->split_labels_into_scored_units(output.tokens); + // remove term insertion weight + approx_ctc -= words.size() * ext_scorer_->beta; + // remove language model weight approx_ctc -= (ext_scorer_->get_sent_log_prob(words)) * ext_scorer_->alpha; } - prefixes_copy[i]->approx_ctc = approx_ctc; + output.confidence = -approx_ctc; + outputs.push_back(output); } - return get_beam_search_result(prefixes_copy, top_paths); + return outputs; } std::vector ctc_beam_search_decoder( diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index 445bfdb2..ed244c3a 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -38,21 +38,6 @@ std::vector> get_pruned_log_probs( return log_prob_idx; } - -std::vector get_beam_search_result( - const std::vector &prefixes, - size_t top_paths) { - std::vector output_vecs; - for (size_t i = 0; i < top_paths && i < prefixes.size(); ++i) { - Output output; - prefixes[i]->get_path_vec(output.tokens, output.timesteps); - output.confidence = -prefixes[i]->approx_ctc; - output_vecs.push_back(output); - } - - return output_vecs; -} - size_t get_utf8_str_len(const std::string &str) { size_t str_len = 0; for (char c : str) { @@ -61,13 +46,12 @@ size_t get_utf8_str_len(const std::string &str) { return str_len; } -std::vector split_utf8_str(const std::string &str) { +std::vector split_into_codepoints(const std::string &str) { std::vector result; std::string out_str; for (char c : str) { - if ((c & 0xc0) != 0x80) // new UTF-8 character - { + if (byte_is_codepoint_boundary(c)) { if (!out_str.empty()) { result.push_back(out_str); out_str.clear(); @@ -80,6 +64,17 @@ std::vector split_utf8_str(const std::string &str) { return result; } +std::vector split_into_bytes(const std::string &str) { + std::vector result; + + for (char c : str) { + std::string ch(1, c); + result.push_back(ch); + } + + return result; +} + std::vector split_str(const std::string &s, const std::string &delim) { std::vector result; @@ -144,27 +139,23 @@ void add_word_to_fst(const std::vector &word, bool add_word_to_dictionary( const std::string &word, const std::unordered_map &char_map, - bool add_space, + bool utf8, int SPACE_ID, fst::StdVectorFst *dictionary) { - auto characters = split_utf8_str(word); + auto characters = utf8 ? split_into_bytes(word) : split_into_codepoints(word); std::vector int_word; for (auto &c : characters) { - if (c == " ") { - int_word.push_back(SPACE_ID); + auto int_c = char_map.find(c); + if (int_c != char_map.end()) { + int_word.push_back(int_c->second); } else { - auto int_c = char_map.find(c); - if (int_c != char_map.end()) { - int_word.push_back(int_c->second); - } else { - return false; // return without adding - } + return false; // return without adding } } - if (add_space) { + if (!utf8) { int_word.push_back(SPACE_ID); } diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index 797810b7..3ba1d7e6 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -59,11 +59,6 @@ std::vector> get_pruned_log_probs( double cutoff_prob, size_t cutoff_top_n); -// Get beam search result from prefixes in trie tree -std::vector get_beam_search_result( - const std::vector &prefixes, - size_t top_paths); - // Functor for prefix comparsion bool prefix_compare(const PathTrie *x, const PathTrie *y); @@ -81,20 +76,30 @@ size_t get_utf8_str_len(const std::string &str); std::vector split_str(const std::string &s, const std::string &delim); -/* Splits string into vector of strings representing - * UTF-8 characters (not same as chars) +/* Splits string into vector of UTF-8 byte sequences representing a single + * Unicode codepoint. */ -std::vector split_utf8_str(const std::string &str); +std::vector split_into_codepoints(const std::string &str); + +/* Splits string into bytes. + */ +std::vector split_into_bytes(const std::string &str); // Add a word in index to the dicionary of fst void add_word_to_fst(const std::vector &word, fst::StdVectorFst *dictionary); +// Return whether a byte is a code point boundary (not a continuation byte). +inline bool byte_is_codepoint_boundary(unsigned char c) { + // only continuation bytes have their most significant bits set to 10 + return (c & 0xC0) != 0x80; +} + // Add a word in string to dictionary bool add_word_to_dictionary( const std::string &word, const std::unordered_map &char_map, - bool add_space, + bool utf8, int SPACE_ID, fst::StdVectorFst *dictionary); #endif // DECODER_UTILS_H diff --git a/native_client/ctcdecode/path_trie.cpp b/native_client/ctcdecode/path_trie.cpp index c1ad2441..749b5a31 100644 --- a/native_client/ctcdecode/path_trie.cpp +++ b/native_client/ctcdecode/path_trie.cpp @@ -111,25 +111,68 @@ PathTrie* PathTrie::get_path_trie(int new_char, int new_timestep, float cur_log_ } } -PathTrie* PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { - return get_path_vec(output, timesteps, ROOT_); -} - -PathTrie* PathTrie::get_path_vec(std::vector& output, - std::vector& timesteps, - int stop, - size_t max_steps) { - if (character == stop || character == ROOT_ || output.size() == max_steps) { - std::reverse(output.begin(), output.end()); - std::reverse(timesteps.begin(), timesteps.end()); - return this; - } else { +void PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + if (parent != nullptr) { + parent->get_path_vec(output, timesteps); + } + if (character != ROOT_) { output.push_back(character); timesteps.push_back(timestep); - return parent->get_path_vec(output, timesteps, stop, max_steps); } } +PathTrie* PathTrie::get_prev_grapheme(std::vector& output, + std::vector& timesteps) +{ + PathTrie* stop = this; + if (character == ROOT_) { + return stop; + } + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + //FIXME: use Alphabet instead of hardcoding +1 here + if (!byte_is_codepoint_boundary(character + 1)) { + stop = parent->get_prev_grapheme(output, timesteps); + } + output.push_back(character); + timesteps.push_back(timestep); + return stop; +} + +int PathTrie::distance_to_codepoint_boundary(unsigned char *first_byte) +{ + //FIXME: use Alphabet instead of hardcoding +1 here + if (byte_is_codepoint_boundary(character + 1)) { + *first_byte = (unsigned char)character + 1; + return 1; + } + if (parent != nullptr && parent->character != ROOT_) { + return 1 + parent->distance_to_codepoint_boundary(first_byte); + } + assert(false); // unreachable + return 0; +} + +PathTrie* PathTrie::get_prev_word(std::vector& output, + std::vector& timesteps, + int space_id) +{ + PathTrie* stop = this; + if (character == space_id || character == ROOT_) { + return stop; + } + // Recursive call: recurse back until stop condition, then append data in + // correct order as we walk back down the stack in the lines below. + if (parent != nullptr) { + stop = parent->get_prev_word(output, timesteps, space_id); + } + output.push_back(character); + timesteps.push_back(timestep); + return stop; +} + void PathTrie::iterate_to_vec(std::vector& output) { if (exists_) { log_prob_b_prev = log_prob_b_cur; @@ -189,7 +232,7 @@ void PathTrie::print(const Alphabet& a) { std::string tr; printf("characters:\t "); for (PathTrie* el : chain) { - printf("%X ", el->character); + printf("%X ", (unsigned char)(el->character)); if (el->character != ROOT_) { tr.append(a.StringFromLabel(el->character)); } diff --git a/native_client/ctcdecode/path_trie.h b/native_client/ctcdecode/path_trie.h index 04cbfdd5..e087b1a6 100644 --- a/native_client/ctcdecode/path_trie.h +++ b/native_client/ctcdecode/path_trie.h @@ -26,14 +26,20 @@ public: // get new prefix after appending new char PathTrie* get_path_trie(int new_char, int new_timestep, float log_prob_c, bool reset = true); - // get the prefix in index from root to current node - PathTrie* get_path_vec(std::vector& output, std::vector& timesteps); + // get the prefix data in correct time order from root to current node + void get_path_vec(std::vector& output, std::vector& timesteps); - // get the prefix in index from some stop node to current nodel - PathTrie* get_path_vec(std::vector& output, - std::vector& timesteps, - int stop, - size_t max_steps = std::numeric_limits::max()); + // get the prefix data in correct time order from beginning of last grapheme to current node + PathTrie* get_prev_grapheme(std::vector& output, + std::vector& timesteps); + + // get the distance from current node to the first codepoint boundary, and the byte value at the boundary + int distance_to_codepoint_boundary(unsigned char *first_byte); + + // get the prefix data in correct time order from beginning of last word to current node + PathTrie* get_prev_word(std::vector& output, + std::vector& timesteps, + int space_id); // update log probs void iterate_to_vec(std::vector& output); diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index c265b430..82197c79 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -27,7 +27,7 @@ using namespace lm::ngram; static const int32_t MAGIC = 'TRIE'; -static const int32_t FILE_VERSION = 4; +static const int32_t FILE_VERSION = 5; int Scorer::init(double alpha, @@ -86,16 +86,21 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path) language_model_.reset(lm::ngram::LoadVirtual(filename, config)); auto vocab = enumerate.vocabulary; for (size_t i = 0; i < vocab.size(); ++i) { - if (is_character_based_ && vocab[i] != UNK_TOKEN && - vocab[i] != START_TOKEN && vocab[i] != END_TOKEN && - get_utf8_str_len(enumerate.vocabulary[i]) > 1) { - is_character_based_ = false; + if (vocab[i] != UNK_TOKEN && + vocab[i] != START_TOKEN && + vocab[i] != END_TOKEN && + get_utf8_str_len(vocab[i]) > 1) { + is_utf8_mode_ = false; + break; } } - // fill the dictionary for FST - if (!is_character_based()) { - fill_dictionary(vocab, true); + + if (alphabet_.GetSize() != 255) { + is_utf8_mode_ = false; } + + // Add spaces only in word-based scoring + fill_dictionary(vocab); } else { config.load_method = util::LoadMethod::LAZY; language_model_.reset(lm::ngram::LoadVirtual(filename, config)); @@ -121,14 +126,12 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path) throw 1; } - fin.read(reinterpret_cast(&is_character_based_), sizeof(is_character_based_)); + fin.read(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); - if (!is_character_based_) { - fst::FstReadOptions opt; - opt.mode = fst::FstReadOptions::MAP; - opt.source = trie_path; - dictionary.reset(FstType::Read(fin, opt)); - } + fst::FstReadOptions opt; + opt.mode = fst::FstReadOptions::MAP; + opt.source = trie_path; + dictionary.reset(FstType::Read(fin, opt)); } max_order_ = language_model_->Order(); @@ -139,12 +142,37 @@ void Scorer::save_dictionary(const std::string& path) std::ofstream fout(path, std::ios::binary); fout.write(reinterpret_cast(&MAGIC), sizeof(MAGIC)); fout.write(reinterpret_cast(&FILE_VERSION), sizeof(FILE_VERSION)); - fout.write(reinterpret_cast(&is_character_based_), sizeof(is_character_based_)); - if (!is_character_based_) { - fst::FstWriteOptions opt; - opt.align = true; - opt.source = path; - dictionary->Write(fout, opt); + fout.write(reinterpret_cast(&is_utf8_mode_), sizeof(is_utf8_mode_)); + fst::FstWriteOptions opt; + opt.align = true; + opt.source = path; + dictionary->Write(fout, opt); +} + +bool Scorer::is_scoring_boundary(PathTrie* prefix, size_t new_label) +{ + if (is_utf8_mode()) { + if (prefix->character == -1) { + return false; + } + unsigned char first_byte; + int distance_to_boundary = prefix->distance_to_codepoint_boundary(&first_byte); + int needed_bytes; + if ((first_byte >> 3) == 0x1E) { + needed_bytes = 4; + } else if ((first_byte >> 4) == 0x0E) { + needed_bytes = 3; + } else if ((first_byte >> 5) == 0x06) { + needed_bytes = 2; + } else if ((first_byte >> 7) == 0x00) { + needed_bytes = 1; + } else { + assert(false); // invalid byte sequence. should be unreachable, disallowed by vocabulary/trie + return false; + } + return distance_to_boundary == needed_bytes; + } else { + return new_label == SPACE_ID_; } } @@ -245,14 +273,14 @@ void Scorer::reset_params(float alpha, float beta) this->beta = beta; } -std::vector Scorer::split_labels(const std::vector& labels) +std::vector Scorer::split_labels_into_scored_units(const std::vector& labels) { if (labels.empty()) return {}; std::string s = alphabet_.LabelsToString(labels); std::vector words; - if (is_character_based_) { - words = split_utf8_str(s); + if (is_utf8_mode_) { + words = split_into_codepoints(s); } else { words = split_str(s, " "); } @@ -266,30 +294,29 @@ std::vector Scorer::make_ngram(PathTrie* prefix) PathTrie* new_node = nullptr; for (int order = 0; order < max_order_; order++) { + if (!current_node || current_node->character == -1) { + break; + } + std::vector prefix_vec; std::vector prefix_steps; - if (is_character_based_) { - new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_, 1); - current_node = new_node; + if (is_utf8_mode_) { + new_node = current_node->get_prev_grapheme(prefix_vec, prefix_steps); } else { - new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_); - current_node = new_node->parent; // Skipping spaces + new_node = current_node->get_prev_word(prefix_vec, prefix_steps, SPACE_ID_); } + current_node = new_node->parent; // reconstruct word std::string word = alphabet_.LabelsToString(prefix_vec); ngram.push_back(word); - - if (new_node->character == -1) { - break; - } } std::reverse(ngram.begin(), ngram.end()); return ngram; } -void Scorer::fill_dictionary(const std::vector& vocabulary, bool add_space) +void Scorer::fill_dictionary(const std::vector& vocabulary) { // ConstFst is immutable, so we need to use a MutableFst to create the trie, // and then we convert to a ConstFst for the decoder and for storing on disk. @@ -297,7 +324,7 @@ void Scorer::fill_dictionary(const std::vector& vocabulary, bool ad // For each unigram convert to ints and put in trie for (const auto& word : vocabulary) { if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) { - add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary); + add_word_to_dictionary(word, char_map_, is_utf8_mode_, SPACE_ID_ + 1, &dictionary); } } diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index 5540138c..96c092b0 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -77,7 +77,7 @@ public: size_t get_max_order() const { return max_order_; } // retrun true if the language model is character based - bool is_character_based() const { return is_character_based_; } + bool is_utf8_mode() const { return is_utf8_mode_; } // reset params alpha & beta void reset_params(float alpha, float beta); @@ -87,11 +87,14 @@ public: // trransform the labels in index to the vector of words (word based lm) or // the vector of characters (character based lm) - std::vector split_labels(const std::vector &labels); + std::vector split_labels_into_scored_units(const std::vector &labels); // save dictionary in file void save_dictionary(const std::string &path); + // return weather this step represents a boundary where beam scoring should happen + bool is_scoring_boundary(PathTrie* prefix, size_t new_label); + // language model weight double alpha = 0.; // word insertion weight @@ -108,11 +111,11 @@ protected: void load_lm(const std::string &lm_path); // fill dictionary for FST - void fill_dictionary(const std::vector &vocabulary, bool add_space); + void fill_dictionary(const std::vector &vocabulary); private: std::unique_ptr language_model_; - bool is_character_based_ = true; + bool is_utf8_mode_ = true; size_t max_order_ = 0; int SPACE_ID_; diff --git a/taskcluster/examples-base.tyml b/taskcluster/examples-base.tyml index 4e7c24e1..8e149a9d 100644 --- a/taskcluster/examples-base.tyml +++ b/taskcluster/examples-base.tyml @@ -30,7 +30,7 @@ then: image: ${build.docker_image} env: - DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.11/models.tar.gz" + DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.14/models.tar.gz" DEEPSPEECH_AUDIO: "https://github.com/mozilla/DeepSpeech/releases/download/v0.4.1/audio-0.4.1.tar.gz" PIP_DEFAULT_TIMEOUT: "60" diff --git a/util/config.py b/util/config.py index fe58e31b..eab52ecb 100755 --- a/util/config.py +++ b/util/config.py @@ -10,7 +10,7 @@ from xdg import BaseDirectory as xdg from util.flags import FLAGS from util.gpu import get_available_gpus from util.logging import log_error -from util.text import Alphabet +from util.text import Alphabet, UTF8Alphabet class ConfigSingleton: _config = None @@ -63,7 +63,10 @@ def initialize_globals(): if not c.available_devices: c.available_devices = [c.cpu_device] - c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path)) + if FLAGS.utf8: + c.alphabet = UTF8Alphabet() + else: + c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path)) # Geometric Constants # =================== diff --git a/util/evaluate_tools.py b/util/evaluate_tools.py index ed69578a..7f6a8ffb 100644 --- a/util/evaluate_tools.py +++ b/util/evaluate_tools.py @@ -6,6 +6,7 @@ from multiprocessing.dummy import Pool from attrdict import AttrDict +from util.flags import FLAGS from util.text import levenshtein @@ -67,7 +68,10 @@ def calculate_report(wav_filenames, labels, decodings, losses): # Order the remaining items by their loss (lowest loss on top) samples.sort(key=lambda s: s.loss) - # Then order by WER (highest WER on top) - samples.sort(key=lambda s: s.wer, reverse=True) + # Then order by descending WER/CER + if FLAGS.utf8: + samples.sort(key=lambda s: s.cer, reverse=True) + else: + samples.sort(key=lambda s: s.wer, reverse=True) return samples_wer, samples_cer, samples diff --git a/util/flags.py b/util/flags.py index 6a8caf3b..a2d2cf2a 100644 --- a/util/flags.py +++ b/util/flags.py @@ -132,6 +132,7 @@ def create_flags(): # Decoder + f.DEFINE_boolean('utf8', False, 'enable UTF-8 mode. When this is used the model outputs UTF-8 sequences directly rather than using an alphabet mapping.') f.DEFINE_string('alphabet_config_path', 'data/alphabet.txt', 'path to the configuration file specifying the alphabet used by the network. See the comment in data/alphabet.txt for a description of the format.') f.DEFINE_string('lm_binary_path', 'data/lm/lm.binary', 'path to the language model binary file created with KenLM') f.DEFINE_alias('lm', 'lm_binary_path') diff --git a/util/text.py b/util/text.py index 7928a990..d3be7eb8 100644 --- a/util/text.py +++ b/util/text.py @@ -76,6 +76,51 @@ class Alphabet(object): return self._config_file +class UTF8Alphabet(object): + @staticmethod + def _string_from_label(_): + assert False + + @staticmethod + def _label_from_string(_): + assert False + + @staticmethod + def encode(string): + # 0 never happens in the data, so we can shift values by one, use 255 for + # the CTC blank, and keep the alphabet size = 256 + return np.frombuffer(string.encode('utf-8'), np.uint8).astype(np.int32) - 1 + + @staticmethod + def decode(labels): + # And here we need to shift back up + return bytes(np.asarray(labels, np.uint8) + 1).decode('utf-8', errors='replace') + + @staticmethod + def size(): + return 255 + + @staticmethod + def serialize(): + res = bytearray() + res += struct.pack('