diff --git a/native_client/deepspeech.cc b/native_client/deepspeech.cc index d9c9e1df..96247615 100644 --- a/native_client/deepspeech.cc +++ b/native_client/deepspeech.cc @@ -115,7 +115,9 @@ struct StreamingState { void feedAudioContent(const short* buffer, unsigned int buffer_size); char* intermediateDecode(); + void finalizeStream(); char* finishStream(); + Metadata* finishStreamWithMetadata(); void processAudioWindow(const vector& buf); void processMfccWindow(const vector& buf); @@ -170,6 +172,28 @@ struct ModelState { */ char* decode(vector& logits); + /** + * @brief Perform decoding of the logits, using basic CTC decoder or + * CTC decoder with KenLM enabled + * + * @param logits Flat matrix of logits, of size: + * n_frames * batch_size * num_classes + * + * @return Vector of Output structs directly from the CTC decoder for additional processing. + */ + vector decode_raw(vector& logits); + + /** + * @brief Return character-level metadata including letter timings. + * + * @param logits Flat matrix of logits, of size: + * n_frames * batch_size * num_classes + * + * @return Metadata struct containing MetadataItem structs for each character. + * The user is responsible for freeing Metadata and Metadata.items. + */ + Metadata* decode_metadata(vector& logits); + /** * @brief Do a single inference step in the acoustic model, with: * input=mfcc @@ -183,6 +207,9 @@ struct ModelState { void infer(const float* mfcc, unsigned int n_frames, vector& output_logits); }; +StreamingState* setupStreamAndFeedAudioContent(ModelState* aCtx, const short* aBuffer, + unsigned int aBufferSize, unsigned int aSampleRate); + ModelState::ModelState() : #ifndef USE_TFLITE @@ -260,22 +287,19 @@ StreamingState::intermediateDecode() char* StreamingState::finishStream() { - // Flush audio buffer - processAudioWindow(audio_buffer); - - // Add empty mfcc vectors at end of sample - for (int i = 0; i < model->n_context; ++i) { - addZeroMfccWindow(); - } - - // Process final batch - if (batch_buffer.size() > 0) { - processBatch(batch_buffer, batch_buffer.size()/model->mfcc_feats_per_timestep); - } + finalizeStream(); return model->decode(accumulated_logits); } +Metadata* +StreamingState::finishStreamWithMetadata() +{ + finalizeStream(); + + return model->decode_metadata(accumulated_logits); +} + void StreamingState::processAudioWindow(const vector& buf) { @@ -291,6 +315,23 @@ StreamingState::processAudioWindow(const vector& buf) free(mfcc); } +void +StreamingState::finalizeStream() +{ + // Flush audio buffer + processAudioWindow(audio_buffer); + + // Add empty mfcc vectors at end of sample + for (int i = 0; i < model->n_context; ++i) { + addZeroMfccWindow(); + } + + // Process final batch + if (batch_buffer.size() > 0) { + processBatch(batch_buffer, batch_buffer.size()/model->mfcc_feats_per_timestep); + } +} + void StreamingState::addZeroMfccWindow() { @@ -415,6 +456,14 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector& logi char* ModelState::decode(vector& logits) +{ + vector out = ModelState::decode_raw(logits); + + return strdup(alphabet->LabelsToString(out[0].tokens).c_str()); +} + +vector +ModelState::decode_raw(vector& logits) { const int cutoff_top_n = 40; const double cutoff_prob = 1.0; @@ -429,7 +478,37 @@ ModelState::decode(vector& logits) inputs.data(), n_frames, num_classes, *alphabet, beam_width, cutoff_prob, cutoff_top_n, scorer); - return strdup(alphabet->LabelsToString(out[0].tokens).c_str()); + return out; +} + +Metadata* ModelState::decode_metadata(vector& logits) +{ + vector out = decode_raw(logits); + + Metadata* metadata = (Metadata*)malloc(sizeof (Metadata)); + metadata->num_items = out[0].tokens.size(); + metadata->items = (MetadataItem*)malloc(sizeof(MetadataItem) * metadata->num_items); + + // Loop through each character + for (int i = 0; i < out[0].tokens.size(); ++i) { + char* character = (char*)alphabet->StringFromLabel(out[0].tokens[i]).c_str(); + + // Note: 1 timestep = 20ms + float start_time = static_cast(out[0].timesteps[i] * AUDIO_WIN_STEP); + + MetadataItem item; + item.character = character; + item.timestep = out[0].timesteps[i]; + item.start_time = start_time; + + if (item.start_time < 0) { + item.start_time = 0; + } + + metadata->items[i] = item; + } + + return metadata; } #ifdef USE_TFLITE @@ -660,14 +739,36 @@ DS_SpeechToText(ModelState* aCtx, const short* aBuffer, unsigned int aBufferSize, unsigned int aSampleRate) +{ + StreamingState* ctx = setupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); + return DS_FinishStream(ctx); +} + +Metadata* +DS_SpeechToTextWithMetadata(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate) +{ + StreamingState* ctx = setupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); + return DS_FinishStreamWithMetadata(ctx); +} + +StreamingState* +setupStreamAndFeedAudioContent(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate) { StreamingState* ctx; int status = DS_SetupStream(aCtx, 0, aSampleRate, &ctx); if (status != DS_ERR_OK) { return nullptr; } + DS_FeedAudioContent(ctx, aBuffer, aBufferSize); - return DS_FinishStream(ctx); + + return ctx; } int @@ -735,6 +836,14 @@ DS_FinishStream(StreamingState* aSctx) return str; } +Metadata* +DS_FinishStreamWithMetadata(StreamingState* aSctx) +{ + Metadata* metadata = aSctx->finishStreamWithMetadata(); + DS_DiscardStream(aSctx); + return metadata; +} + void DS_DiscardStream(StreamingState* aSctx) { @@ -809,6 +918,13 @@ DS_AudioToInputVector(const short* aBuffer, } } +void +DS_FreeMetadata(Metadata* m) +{ + free(m->items); + free(m); +} + void DS_PrintVersions() { std::cerr << "TensorFlow: " << tf_local_git_version() << std::endl; diff --git a/native_client/deepspeech.h b/native_client/deepspeech.h index 77b2b43b..e0d52d94 100644 --- a/native_client/deepspeech.h +++ b/native_client/deepspeech.h @@ -15,6 +15,19 @@ struct ModelState; struct StreamingState; +// Stores each individual character, along with its timing information +struct MetadataItem { + char* character; + int timestep; // Position of the character in units of 20ms + float start_time; // Position of the character in seconds +}; + +// Stores the entire CTC output as an array of character metadata objects +struct Metadata { + MetadataItem* items; + int num_items; +}; + enum DeepSpeech_Error_Codes { // OK @@ -109,6 +122,25 @@ char* DS_SpeechToText(ModelState* aCtx, unsigned int aBufferSize, unsigned int aSampleRate); +/** + * @brief Use the DeepSpeech model to perform Speech-To-Text and output metadata + * about the results. + * + * @param aCtx The ModelState pointer for the model to use. + * @param aBuffer A 16-bit, mono raw audio signal at the appropriate + * sample rate. + * @param aBufferSize The number of samples in the audio signal. + * @param aSampleRate The sample-rate of the audio signal. + * + * @return Outputs a struct of individual letters along with their timing information. + * The user is responsible for freeing Metadata and Metadata.items. Returns NULL on error. + */ +DEEPSPEECH_EXPORT +Metadata* DS_SpeechToTextWithMetadata(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate); + /** * @brief Create a new streaming inference state. The streaming state returned * by this function can then be passed to {@link DS_FeedAudioContent()} @@ -170,6 +202,20 @@ char* DS_IntermediateDecode(StreamingState* aSctx); DEEPSPEECH_EXPORT char* DS_FinishStream(StreamingState* aSctx); +/** + * @brief Signal the end of an audio signal to an ongoing streaming + * inference, returns per-letter metadata. + * + * @param aSctx A streaming state pointer returned by {@link DS_SetupStream()}. + * + * @return Outputs a struct of individual letters along with their timing information. + * The user is responsible for freeing Metadata and Metadata.items. Returns NULL on error. + * + * @note This method will free the state pointer (@p aSctx). + */ +DEEPSPEECH_EXPORT +Metadata* DS_FinishStreamWithMetadata(StreamingState* aSctx); + /** * @brief Destroy a streaming state without decoding the computed logits. This * can be used if you no longer need the result of an ongoing streaming @@ -213,6 +259,13 @@ void DS_AudioToInputVector(const short* aBuffer, int* aNFrames = NULL, int* aFrameLen = NULL); +/** + * @brief Free memory allocated for metadata information. + */ + +DEEPSPEECH_EXPORT +void DS_FreeMetadata(Metadata* m); + /** * @brief Print version of this library and of the linked TensorFlow library. */