#ifndef ALPHABET_H #define ALPHABET_H #include #include #include /* * Loads a text file describing a mapping of labels to strings, one string per * line. This is used by the decoder, client and Python scripts to convert the * output of the decoder to a human-readable string and vice-versa. */ class Alphabet { public: Alphabet() = default; Alphabet(const Alphabet&) = default; Alphabet& operator=(const Alphabet&) = default; virtual ~Alphabet() = default; virtual int init(const char *config_file); // Serialize alphabet into a binary buffer. std::string Serialize(); // Deserialize alphabet from a binary buffer. int Deserialize(const char* buffer, const int buffer_size); size_t GetSize() const { return size_; } bool IsSpace(unsigned int label) const { return label == space_label_; } unsigned int GetSpaceLabel() const { return space_label_; } // Decode a single label into a string. std::string DecodeSingle(unsigned int label) const; // Encode a single character/output class into a label. unsigned int EncodeSingle(const std::string& string) const; // Decode a sequence of labels into a string. std::string Decode(const std::vector& input) const; // We provide a C-style overload for accepting NumPy arrays as input, since // the NumPy library does not have built-in typemaps for std::vector. std::string Decode(const unsigned int* input, int length) const; // Encode a sequence of character/output classes into a sequence of labels. // Characters are assumed to always take a single Unicode codepoint. std::vector Encode(const std::string& input) const; protected: size_t size_; unsigned int space_label_; std::unordered_map label_to_str_; std::unordered_map str_to_label_; }; class UTF8Alphabet : public Alphabet { public: UTF8Alphabet() { size_ = 255; space_label_ = ' ' - 1; for (size_t i = 0; i < size_; ++i) { std::string val(1, i+1); label_to_str_[i] = val; str_to_label_[val] = i; } } int init(const char*) override { return 0; } }; #endif //ALPHABET_H