mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2521 from mozilla/utf8

Browse Source 
UTF-8 target
This commit is contained in:
Reuben Morais 2019-11-13 11:24:45 +00:00 committed by GitHub
commit 1eaec6eb5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 267 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
data/lm/trie (Stored with Git LFS)
View File

Binary file not shown.

BIN
data/smoke_test/vocab.trie
View File

Binary file not shown.

62
native_client/ctcdecode/ctc_beam_search_decoder.cpp
View File

@ -36,7 +36,7 @@ DecoderState::init(const Alphabet& alphabet,
prefix_root_.reset(root);
prefixes_.push_back(root);
if (ext_scorer != nullptr && !ext_scorer->is_character_based()) {
if (ext_scorer != nullptr) {
// no need for std::make_shared<>() since Copy() does 'new' behind the doors
auto dict_ptr = std::shared_ptr<PathTrie::FstType>(ext_scorer->dictionary->Copy(true));
root->set_dictionary(dict_ptr);
@ -109,24 +109,25 @@ DecoderState::next(const double *probs,
log_p = log_prob_c + prefix->score;
}
// language model scoring
if (ext_scorer_ != nullptr &&
(c == space_id_ || ext_scorer_->is_character_based())) {
PathTrie *prefix_to_score = nullptr;
// skip scoring the space
if (ext_scorer_->is_character_based()) {
if (ext_scorer_ != nullptr) {
// skip scoring the space in word based LMs
PathTrie* prefix_to_score;
if (ext_scorer_->is_utf8_mode()) {
prefix_to_score = prefix_new;
} else {
prefix_to_score = prefix;
}
float score = 0.0;
std::vector<std::string> ngram;
ngram = ext_scorer_->make_ngram(prefix_to_score);
bool bos = ngram.size() < ext_scorer_->get_max_order();
score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha;
log_p += score;
log_p += ext_scorer_->beta;
// language model scoring
if (ext_scorer_->is_scoring_boundary(prefix_to_score, c)) {
float score = 0.0;
std::vector<std::string> ngram;
ngram = ext_scorer_->make_ngram(prefix_to_score);
bool bos = ngram.size() < ext_scorer_->get_max_order();
score = ext_scorer_->get_log_cond_prob(ngram, bos) * ext_scorer_->alpha;
log_p += score;
log_p += ext_scorer_->beta;
}
}
prefix_new->log_prob_nb_cur =
@ -165,10 +166,12 @@ DecoderState::decode() const
}
// score the last word of each prefix that doesn't end with space
if (ext_scorer_ != nullptr && !ext_scorer_->is_character_based()) {
if (ext_scorer_ != nullptr) {
for (size_t i = 0; i < beam_size_ && i < prefixes_copy.size(); ++i) {
auto prefix = prefixes_copy[i];
if (!prefix->is_empty() && prefix->character != space_id_) {
if (prefix->is_empty()) {
scores[prefix] = OOV_SCORE;
} else if (!ext_scorer_->is_scoring_boundary(prefix->parent, prefix->character)) {
float score = 0.0;
std::vector<std::string> ngram = ext_scorer_->make_ngram(prefix);
bool bos = ngram.size() < ext_scorer_->get_max_order();
@ -187,27 +190,30 @@ DecoderState::decode() const
std::bind(prefix_compare_external, _1, _2, scores));
//TODO: expose this as an API parameter
const int top_paths = 1;
const size_t top_paths = 1;
size_t num_returned = std::min(num_prefixes, top_paths);
std::vector<Output> outputs;
outputs.reserve(num_returned);
// compute aproximate ctc score as the return score, without affecting the
// return order of decoding result. To delete when decoder gets stable.
for (size_t i = 0; i < top_paths && i < prefixes_copy.size(); ++i) {
for (size_t i = 0; i < num_returned; ++i) {
Output output;
prefixes_copy[i]->get_path_vec(output.tokens, output.timesteps);
double approx_ctc = scores[prefixes_copy[i]];
if (ext_scorer_ != nullptr) {
std::vector<int> output;
std::vector<int> timesteps;
prefixes_copy[i]->get_path_vec(output, timesteps);
auto prefix_length = output.size();
auto words = ext_scorer_->split_labels(output);
// remove word insert
approx_ctc = approx_ctc - prefix_length * ext_scorer_->beta;
// remove language model weight:
auto words = ext_scorer_->split_labels_into_scored_units(output.tokens);
// remove term insertion weight
approx_ctc -= words.size() * ext_scorer_->beta;
// remove language model weight
approx_ctc -= (ext_scorer_->get_sent_log_prob(words)) * ext_scorer_->alpha;
}
prefixes_copy[i]->approx_ctc = approx_ctc;
output.confidence = -approx_ctc;
outputs.push_back(output);
}
return get_beam_search_result(prefixes_copy, top_paths);
return outputs;
}
std::vector<Output> ctc_beam_search_decoder(

49
native_client/ctcdecode/decoder_utils.cpp
View File

@ -38,21 +38,6 @@ std::vector<std::pair<size_t, float>> get_pruned_log_probs(
return log_prob_idx;
}
std::vector<Output> get_beam_search_result(
const std::vector<PathTrie *> &prefixes,
size_t top_paths) {
std::vector<Output> output_vecs;
for (size_t i = 0; i < top_paths && i < prefixes.size(); ++i) {
Output output;
prefixes[i]->get_path_vec(output.tokens, output.timesteps);
output.confidence = -prefixes[i]->approx_ctc;
output_vecs.push_back(output);
}
return output_vecs;
}
size_t get_utf8_str_len(const std::string &str) {
size_t str_len = 0;
for (char c : str) {
@ -61,13 +46,12 @@ size_t get_utf8_str_len(const std::string &str) {
return str_len;
}
std::vector<std::string> split_utf8_str(const std::string &str) {
std::vector<std::string> split_into_codepoints(const std::string &str) {
std::vector<std::string> result;
std::string out_str;
for (char c : str) {
if ((c & 0xc0) != 0x80) // new UTF-8 character
{
if (byte_is_codepoint_boundary(c)) {
if (!out_str.empty()) {
result.push_back(out_str);
out_str.clear();
@ -80,6 +64,17 @@ std::vector<std::string> split_utf8_str(const std::string &str) {
return result;
}
std::vector<std::string> split_into_bytes(const std::string &str) {
std::vector<std::string> result;
for (char c : str) {
std::string ch(1, c);
result.push_back(ch);
}
return result;
}
std::vector<std::string> split_str(const std::string &s,
const std::string &delim) {
std::vector<std::string> result;
@ -144,27 +139,23 @@ void add_word_to_fst(const std::vector<int> &word,
bool add_word_to_dictionary(
const std::string &word,
const std::unordered_map<std::string, int> &char_map,
bool add_space,
bool utf8,
int SPACE_ID,
fst::StdVectorFst *dictionary) {
auto characters = split_utf8_str(word);
auto characters = utf8 ? split_into_bytes(word) : split_into_codepoints(word);
std::vector<int> int_word;
for (auto &c : characters) {
if (c == " ") {
int_word.push_back(SPACE_ID);
auto int_c = char_map.find(c);
if (int_c != char_map.end()) {
int_word.push_back(int_c->second);
} else {
auto int_c = char_map.find(c);
if (int_c != char_map.end()) {
int_word.push_back(int_c->second);
} else {
return false; // return without adding
}
return false; // return without adding
}
}
if (add_space) {
if (!utf8) {
int_word.push_back(SPACE_ID);
}

23
native_client/ctcdecode/decoder_utils.h
View File

@ -59,11 +59,6 @@ std::vector<std::pair<size_t, float>> get_pruned_log_probs(
double cutoff_prob,
size_t cutoff_top_n);
// Get beam search result from prefixes in trie tree
std::vector<Output> get_beam_search_result(
const std::vector<PathTrie *> &prefixes,
size_t top_paths);
// Functor for prefix comparsion
bool prefix_compare(const PathTrie *x, const PathTrie *y);
@ -81,20 +76,30 @@ size_t get_utf8_str_len(const std::string &str);
std::vector<std::string> split_str(const std::string &s,
const std::string &delim);
/* Splits string into vector of strings representing
* UTF-8 characters (not same as chars)
/* Splits string into vector of UTF-8 byte sequences representing a single
* Unicode codepoint.
*/
std::vector<std::string> split_utf8_str(const std::string &str);
std::vector<std::string> split_into_codepoints(const std::string &str);
/* Splits string into bytes.
*/
std::vector<std::string> split_into_bytes(const std::string &str);
// Add a word in index to the dicionary of fst
void add_word_to_fst(const std::vector<int> &word,
fst::StdVectorFst *dictionary);
// Return whether a byte is a code point boundary (not a continuation byte).
inline bool byte_is_codepoint_boundary(unsigned char c) {
// only continuation bytes have their most significant bits set to 10
return (c & 0xC0) != 0x80;
}
// Add a word in string to dictionary
bool add_word_to_dictionary(
const std::string &word,
const std::unordered_map<std::string, int> &char_map,
bool add_space,
bool utf8,
int SPACE_ID,
fst::StdVectorFst *dictionary);
#endif // DECODER_UTILS_H

73
native_client/ctcdecode/path_trie.cpp
View File

@ -111,25 +111,68 @@ PathTrie* PathTrie::get_path_trie(int new_char, int new_timestep, float cur_log_
}
}
PathTrie* PathTrie::get_path_vec(std::vector<int>& output, std::vector<int>& timesteps) {
return get_path_vec(output, timesteps, ROOT_);
}
PathTrie* PathTrie::get_path_vec(std::vector<int>& output,
std::vector<int>& timesteps,
int stop,
size_t max_steps) {
if (character == stop || character == ROOT_ || output.size() == max_steps) {
std::reverse(output.begin(), output.end());
std::reverse(timesteps.begin(), timesteps.end());
return this;
} else {
void PathTrie::get_path_vec(std::vector<int>& output, std::vector<int>& timesteps) {
// Recursive call: recurse back until stop condition, then append data in
// correct order as we walk back down the stack in the lines below.
if (parent != nullptr) {
parent->get_path_vec(output, timesteps);
}
if (character != ROOT_) {
output.push_back(character);
timesteps.push_back(timestep);
return parent->get_path_vec(output, timesteps, stop, max_steps);
}
}
PathTrie* PathTrie::get_prev_grapheme(std::vector<int>& output,
std::vector<int>& timesteps)
{
PathTrie* stop = this;
if (character == ROOT_) {
return stop;
}
// Recursive call: recurse back until stop condition, then append data in
// correct order as we walk back down the stack in the lines below.
//FIXME: use Alphabet instead of hardcoding +1 here
if (!byte_is_codepoint_boundary(character + 1)) {
stop = parent->get_prev_grapheme(output, timesteps);
}
output.push_back(character);
timesteps.push_back(timestep);
return stop;
}
int PathTrie::distance_to_codepoint_boundary(unsigned char *first_byte)
{
//FIXME: use Alphabet instead of hardcoding +1 here
if (byte_is_codepoint_boundary(character + 1)) {
*first_byte = (unsigned char)character + 1;
return 1;
}
if (parent != nullptr && parent->character != ROOT_) {
return 1 + parent->distance_to_codepoint_boundary(first_byte);
}
assert(false); // unreachable
return 0;
}
PathTrie* PathTrie::get_prev_word(std::vector<int>& output,
std::vector<int>& timesteps,
int space_id)
{
PathTrie* stop = this;
if (character == space_id || character == ROOT_) {
return stop;
}
// Recursive call: recurse back until stop condition, then append data in
// correct order as we walk back down the stack in the lines below.
if (parent != nullptr) {
stop = parent->get_prev_word(output, timesteps, space_id);
}
output.push_back(character);
timesteps.push_back(timestep);
return stop;
}
void PathTrie::iterate_to_vec(std::vector<PathTrie*>& output) {
if (exists_) {
log_prob_b_prev = log_prob_b_cur;
@ -189,7 +232,7 @@ void PathTrie::print(const Alphabet& a) {
std::string tr;
printf("characters:\t ");
for (PathTrie* el : chain) {
printf("%X ", el->character);
printf("%X ", (unsigned char)(el->character));
if (el->character != ROOT_) {
tr.append(a.StringFromLabel(el->character));
}

20
native_client/ctcdecode/path_trie.h
View File

@ -26,14 +26,20 @@ public:
// get new prefix after appending new char
PathTrie* get_path_trie(int new_char, int new_timestep, float log_prob_c, bool reset = true);
// get the prefix in index from root to current node
PathTrie* get_path_vec(std::vector<int>& output, std::vector<int>& timesteps);
// get the prefix data in correct time order from root to current node
void get_path_vec(std::vector<int>& output, std::vector<int>& timesteps);
// get the prefix in index from some stop node to current nodel
PathTrie* get_path_vec(std::vector<int>& output,
std::vector<int>& timesteps,
int stop,
size_t max_steps = std::numeric_limits<size_t>::max());
// get the prefix data in correct time order from beginning of last grapheme to current node
PathTrie* get_prev_grapheme(std::vector<int>& output,
std::vector<int>& timesteps);
// get the distance from current node to the first codepoint boundary, and the byte value at the boundary
int distance_to_codepoint_boundary(unsigned char *first_byte);
// get the prefix data in correct time order from beginning of last word to current node
PathTrie* get_prev_word(std::vector<int>& output,
std::vector<int>& timesteps,
int space_id);
// update log probs
void iterate_to_vec(std::vector<PathTrie*>& output);

97
native_client/ctcdecode/scorer.cpp
View File

@ -27,7 +27,7 @@
using namespace lm::ngram;
static const int32_t MAGIC = 'TRIE';
static const int32_t FILE_VERSION = 4;
static const int32_t FILE_VERSION = 5;
int
Scorer::init(double alpha,
@ -86,16 +86,21 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path)
language_model_.reset(lm::ngram::LoadVirtual(filename, config));
auto vocab = enumerate.vocabulary;
for (size_t i = 0; i < vocab.size(); ++i) {
if (is_character_based_ && vocab[i] != UNK_TOKEN &&
vocab[i] != START_TOKEN && vocab[i] != END_TOKEN &&
get_utf8_str_len(enumerate.vocabulary[i]) > 1) {
is_character_based_ = false;
if (vocab[i] != UNK_TOKEN &&
vocab[i] != START_TOKEN &&
vocab[i] != END_TOKEN &&
get_utf8_str_len(vocab[i]) > 1) {
is_utf8_mode_ = false;
break;
}
}
// fill the dictionary for FST
if (!is_character_based()) {
fill_dictionary(vocab, true);
if (alphabet_.GetSize() != 255) {
is_utf8_mode_ = false;
}
// Add spaces only in word-based scoring
fill_dictionary(vocab);
} else {
config.load_method = util::LoadMethod::LAZY;
language_model_.reset(lm::ngram::LoadVirtual(filename, config));
@ -121,14 +126,12 @@ void Scorer::setup(const std::string& lm_path, const std::string& trie_path)
throw 1;
}
fin.read(reinterpret_cast<char*>(&is_character_based_), sizeof(is_character_based_));
fin.read(reinterpret_cast<char*>(&is_utf8_mode_), sizeof(is_utf8_mode_));
if (!is_character_based_) {
fst::FstReadOptions opt;
opt.mode = fst::FstReadOptions::MAP;
opt.source = trie_path;
dictionary.reset(FstType::Read(fin, opt));
}
fst::FstReadOptions opt;
opt.mode = fst::FstReadOptions::MAP;
opt.source = trie_path;
dictionary.reset(FstType::Read(fin, opt));
}
max_order_ = language_model_->Order();
@ -139,12 +142,37 @@ void Scorer::save_dictionary(const std::string& path)
std::ofstream fout(path, std::ios::binary);
fout.write(reinterpret_cast<const char*>(&MAGIC), sizeof(MAGIC));
fout.write(reinterpret_cast<const char*>(&FILE_VERSION), sizeof(FILE_VERSION));
fout.write(reinterpret_cast<const char*>(&is_character_based_), sizeof(is_character_based_));
if (!is_character_based_) {
fst::FstWriteOptions opt;
opt.align = true;
opt.source = path;
dictionary->Write(fout, opt);
fout.write(reinterpret_cast<const char*>(&is_utf8_mode_), sizeof(is_utf8_mode_));
fst::FstWriteOptions opt;
opt.align = true;
opt.source = path;
dictionary->Write(fout, opt);
}
bool Scorer::is_scoring_boundary(PathTrie* prefix, size_t new_label)
{
if (is_utf8_mode()) {
if (prefix->character == -1) {
return false;
}
unsigned char first_byte;
int distance_to_boundary = prefix->distance_to_codepoint_boundary(&first_byte);
int needed_bytes;
if ((first_byte >> 3) == 0x1E) {
needed_bytes = 4;
} else if ((first_byte >> 4) == 0x0E) {
needed_bytes = 3;
} else if ((first_byte >> 5) == 0x06) {
needed_bytes = 2;
} else if ((first_byte >> 7) == 0x00) {
needed_bytes = 1;
} else {
assert(false); // invalid byte sequence. should be unreachable, disallowed by vocabulary/trie
return false;
}
return distance_to_boundary == needed_bytes;
} else {
return new_label == SPACE_ID_;
}
}
@ -245,14 +273,14 @@ void Scorer::reset_params(float alpha, float beta)
this->beta = beta;
}
std::vector<std::string> Scorer::split_labels(const std::vector<int>& labels)
std::vector<std::string> Scorer::split_labels_into_scored_units(const std::vector<int>& labels)
{
if (labels.empty()) return {};
std::string s = alphabet_.LabelsToString(labels);
std::vector<std::string> words;
if (is_character_based_) {
words = split_utf8_str(s);
if (is_utf8_mode_) {
words = split_into_codepoints(s);
} else {
words = split_str(s, " ");
}
@ -266,30 +294,29 @@ std::vector<std::string> Scorer::make_ngram(PathTrie* prefix)
PathTrie* new_node = nullptr;
for (int order = 0; order < max_order_; order++) {
if (!current_node || current_node->character == -1) {
break;
}
std::vector<int> prefix_vec;
std::vector<int> prefix_steps;
if (is_character_based_) {
new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_, 1);
current_node = new_node;
if (is_utf8_mode_) {
new_node = current_node->get_prev_grapheme(prefix_vec, prefix_steps);
} else {
new_node = current_node->get_path_vec(prefix_vec, prefix_steps, SPACE_ID_);
current_node = new_node->parent; // Skipping spaces
new_node = current_node->get_prev_word(prefix_vec, prefix_steps, SPACE_ID_);
}
current_node = new_node->parent;
// reconstruct word
std::string word = alphabet_.LabelsToString(prefix_vec);
ngram.push_back(word);
if (new_node->character == -1) {
break;
}
}
std::reverse(ngram.begin(), ngram.end());
return ngram;
}
void Scorer::fill_dictionary(const std::vector<std::string>& vocabulary, bool add_space)
void Scorer::fill_dictionary(const std::vector<std::string>& vocabulary)
{
// ConstFst is immutable, so we need to use a MutableFst to create the trie,
// and then we convert to a ConstFst for the decoder and for storing on disk.
@ -297,7 +324,7 @@ void Scorer::fill_dictionary(const std::vector<std::string>& vocabulary, bool ad
// For each unigram convert to ints and put in trie
for (const auto& word : vocabulary) {
if (word != START_TOKEN && word != UNK_TOKEN && word != END_TOKEN) {
add_word_to_dictionary(word, char_map_, add_space, SPACE_ID_ + 1, &dictionary);
add_word_to_dictionary(word, char_map_, is_utf8_mode_, SPACE_ID_ + 1, &dictionary);
}
}

11
native_client/ctcdecode/scorer.h
View File

@ -77,7 +77,7 @@ public:
size_t get_max_order() const { return max_order_; }
// retrun true if the language model is character based
bool is_character_based() const { return is_character_based_; }
bool is_utf8_mode() const { return is_utf8_mode_; }
// reset params alpha & beta
void reset_params(float alpha, float beta);
@ -87,11 +87,14 @@ public:
// trransform the labels in index to the vector of words (word based lm) or
// the vector of characters (character based lm)
std::vector<std::string> split_labels(const std::vector<int> &labels);
std::vector<std::string> split_labels_into_scored_units(const std::vector<int> &labels);
// save dictionary in file
void save_dictionary(const std::string &path);
// return weather this step represents a boundary where beam scoring should happen
bool is_scoring_boundary(PathTrie* prefix, size_t new_label);
// language model weight
double alpha = 0.;
// word insertion weight
@ -108,11 +111,11 @@ protected:
void load_lm(const std::string &lm_path);
// fill dictionary for FST
void fill_dictionary(const std::vector<std::string> &vocabulary, bool add_space);
void fill_dictionary(const std::vector<std::string> &vocabulary);
private:
std::unique_ptr<lm::base::Model> language_model_;
bool is_character_based_ = true;
bool is_utf8_mode_ = true;
size_t max_order_ = 0;
int SPACE_ID_;

2
taskcluster/examples-base.tyml
View File

@ -30,7 +30,7 @@ then:
image: ${build.docker_image}
env:
DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.11/models.tar.gz"
DEEPSPEECH_MODEL: "https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.14/models.tar.gz"
DEEPSPEECH_AUDIO: "https://github.com/mozilla/DeepSpeech/releases/download/v0.4.1/audio-0.4.1.tar.gz"
PIP_DEFAULT_TIMEOUT: "60"

7
util/config.py
View File

@ -10,7 +10,7 @@ from xdg import BaseDirectory as xdg
from util.flags import FLAGS
from util.gpu import get_available_gpus
from util.logging import log_error
from util.text import Alphabet
from util.text import Alphabet, UTF8Alphabet
class ConfigSingleton:
_config = None
@ -63,7 +63,10 @@ def initialize_globals():
if not c.available_devices:
c.available_devices = [c.cpu_device]
c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path))
if FLAGS.utf8:
c.alphabet = UTF8Alphabet()
else:
c.alphabet = Alphabet(os.path.abspath(FLAGS.alphabet_config_path))
# Geometric Constants
# ===================

8
util/evaluate_tools.py
View File

@ -6,6 +6,7 @@ from multiprocessing.dummy import Pool
from attrdict import AttrDict
from util.flags import FLAGS
from util.text import levenshtein
@ -67,7 +68,10 @@ def calculate_report(wav_filenames, labels, decodings, losses):
# Order the remaining items by their loss (lowest loss on top)
samples.sort(key=lambda s: s.loss)
# Then order by WER (highest WER on top)
samples.sort(key=lambda s: s.wer, reverse=True)
# Then order by descending WER/CER
if FLAGS.utf8:
samples.sort(key=lambda s: s.cer, reverse=True)
else:
samples.sort(key=lambda s: s.wer, reverse=True)
return samples_wer, samples_cer, samples

1
util/flags.py
View File

@ -132,6 +132,7 @@ def create_flags():
# Decoder
f.DEFINE_boolean('utf8', False, 'enable UTF-8 mode. When this is used the model outputs UTF-8 sequences directly rather than using an alphabet mapping.')
f.DEFINE_string('alphabet_config_path', 'data/alphabet.txt', 'path to the configuration file specifying the alphabet used by the network. See the comment in data/alphabet.txt for a description of the format.')
f.DEFINE_string('lm_binary_path', 'data/lm/lm.binary', 'path to the language model binary file created with KenLM')
f.DEFINE_alias('lm', 'lm_binary_path')

45
util/text.py
View File

@ -76,6 +76,51 @@ class Alphabet(object):
return self._config_file
class UTF8Alphabet(object):
@staticmethod
def _string_from_label(_):
assert False
@staticmethod
def _label_from_string(_):
assert False
@staticmethod
def encode(string):
# 0 never happens in the data, so we can shift values by one, use 255 for
# the CTC blank, and keep the alphabet size = 256
return np.frombuffer(string.encode('utf-8'), np.uint8).astype(np.int32) - 1
@staticmethod
def decode(labels):
# And here we need to shift back up
return bytes(np.asarray(labels, np.uint8) + 1).decode('utf-8', errors='replace')
@staticmethod
def size():
return 255
@staticmethod
def serialize():
res = bytearray()
res += struct.pack('<h', 255)
for i in range(255):
# Note that we also shift back up in the mapping constructed here
# so that the native client sees the correct byte values when decoding.
res += struct.pack('<hh1s', i, 1, bytes([i+1]))
return bytes(res)
@staticmethod
def deserialize(buf):
size = struct.unpack('<I', buf)[0]
assert size == 255
return UTF8Alphabet()
@staticmethod
def config_file():
return ''
def text_to_char_array(series, alphabet):
r"""
Given a Pandas Series containing transcript string, map characters to