mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add math defines Windows

Browse Source
This commit is contained in:
Carlos Fonseca M 2018-12-15 11:43:17 -06:00
parent 714673ed30
commit 17517cb453
2 changed files with 244 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
native_client/BUILD
View File

@ -3,6 +3,9 @@
load("@org_tensorflow//tensorflow:tensorflow.bzl",
"tf_cc_shared_object", "if_cuda")
load("@org_tensorflow//tensorflow/contrib/lite:build_def.bzl",
"tflite_copts", "tflite_linkopts")
genrule(
name = "ds_git_version",
outs = ["ds_version.h"],
@ -67,48 +70,54 @@ tf_cc_shared_object(
# -Wno-sign-compare to silent a lot of warnings from tensorflow itself,
# which makes it harder to see our own warnings
"//conditions:default": ["-Wno-sign-compare", "-fvisibility=hidden"],
}),
}) + tflite_copts(),
linkopts = select({
"//tensorflow:darwin": [],
"//tensorflow:linux_x86_64": LINUX_LINKOPTS,
"//tensorflow:rpi3": LINUX_LINKOPTS + ["-l:libstdc++.a"],
"//tensorflow:rpi3-armv8": LINUX_LINKOPTS + ["-l:libstdc++.a"],
"//tensorflow:windows": [],
}),
deps = [
"//tensorflow/core:core_cpu",
"//tensorflow/core:direct_session",
"//third_party/eigen3",
#"//tensorflow/core:all_kernels",
### => Trying to be more fine-grained
### Use bin/ops_in_graph.py to list all the ops used by a frozen graph.
### CPU only build, libdeepspeech.so file size reduced by ~50%
"//tensorflow/core/kernels:dense_update_ops", # Assign
"//tensorflow/core/kernels:constant_op", # Const
"//tensorflow/core/kernels:immutable_constant_op", # ImmutableConst
"//tensorflow/core/kernels:identity_op", # Identity
"//tensorflow/core/kernels:softmax_op", # Softmax
"//tensorflow/core/kernels:transpose_op", # Transpose
"//tensorflow/core/kernels:reshape_op", # Reshape
"//tensorflow/core/kernels:shape_ops", # Shape
"//tensorflow/core/kernels:concat_op", # ConcatV2
"//tensorflow/core/kernels:split_op", # Split
"//tensorflow/core/kernels:variable_ops", # VariableV2
"//tensorflow/core/kernels:relu_op", # Relu
"//tensorflow/core/kernels:bias_op", # BiasAdd
"//tensorflow/core/kernels:math", # Range, MatMul
"//tensorflow/core/kernels:control_flow_ops", # Enter
"//tensorflow/core/kernels:tile_ops", # Tile
"//tensorflow/core/kernels:gather_op", # Gather
"//tensorflow/contrib/rnn:lstm_ops_kernels", # BlockLSTM
"//tensorflow/core/kernels:random_ops", # RandomGammaGrad
"//tensorflow/core/kernels:pack_op", # Pack
"//tensorflow/core/kernels:gather_nd_op", # GatherNd
#### Needed by production model produced without "--use_seq_length False"
#"//tensorflow/core/kernels:logging_ops", # Assert
#"//tensorflow/core/kernels:reverse_sequence_op", # ReverseSequence
] + if_cuda([
"//tensorflow/core:core",
"//conditions:default": []
}) + tflite_linkopts(),
deps = select({
"//tensorflow:android": [
"//tensorflow/contrib/lite/kernels:builtin_ops",
],
"//conditions:default": [
"//tensorflow/core:core_cpu",
"//tensorflow/core:direct_session",
"//third_party/eigen3",
#"//tensorflow/core:all_kernels",
### => Trying to be more fine-grained
### Use bin/ops_in_graph.py to list all the ops used by a frozen graph.
### CPU only build, libdeepspeech.so file size reduced by ~50%
"//tensorflow/core/kernels:dense_update_ops", # Assign
"//tensorflow/core/kernels:constant_op", # Const
"//tensorflow/core/kernels:immutable_constant_op", # ImmutableConst
"//tensorflow/core/kernels:identity_op", # Identity
"//tensorflow/core/kernels:softmax_op", # Softmax
"//tensorflow/core/kernels:transpose_op", # Transpose
"//tensorflow/core/kernels:reshape_op", # Reshape
"//tensorflow/core/kernels:shape_ops", # Shape
"//tensorflow/core/kernels:concat_op", # ConcatV2
"//tensorflow/core/kernels:split_op", # Split
"//tensorflow/core/kernels:variable_ops", # VariableV2
"//tensorflow/core/kernels:relu_op", # Relu
"//tensorflow/core/kernels:bias_op", # BiasAdd
"//tensorflow/core/kernels:math", # Range, MatMul
"//tensorflow/core/kernels:control_flow_ops", # Enter
"//tensorflow/core/kernels:tile_ops", # Tile
"//tensorflow/core/kernels:gather_op", # Gather
"//tensorflow/contrib/rnn:lstm_ops_kernels", # BlockLSTM
"//tensorflow/core/kernels:random_ops", # RandomGammaGrad
"//tensorflow/core/kernels:pack_op", # Pack
"//tensorflow/core/kernels:gather_nd_op", # GatherNd
#### Needed by production model produced without "--use_seq_length False"
#"//tensorflow/core/kernels:logging_ops", # Assert
#"//tensorflow/core/kernels:reverse_sequence_op", # ReverseSequence
],
}) + if_cuda([
"//tensorflow/core:core",
]),
includes = ["c_speech_features", "kiss_fft130"] + DECODER_INCLUDES,
defines = ["KENLM_MAX_ORDER=6"],

226
native_client/deepspeech.cc
View File

@ -1,4 +1,8 @@
#include <algorithm>
#ifdef _MSC_VER
#define _USE_MATH_DEFINES
#endif
#include <cmath>
#include <iostream>
#include <memory>
#include <string>
@ -8,37 +12,71 @@
#include "deepspeech.h"
#include "alphabet.h"
#ifndef USE_TFLITE
#include "tensorflow/core/public/version.h"
#endif // USE_TFLITE
#include "native_client/ds_version.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/util/memmapped_file_system.h"
#ifndef USE_TFLITE
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/util/memmapped_file_system.h"
#else // USE_TFLITE
#include "tensorflow/contrib/lite/model.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#endif // USE_TFLITE
#include "c_speech_features.h"
#include "ctcdecode/ctc_beam_search_decoder.h"
#ifdef __ANDROID__
#include <android/log.h>
#define LOG_TAG "libdeepspeech"
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
#else
#define LOGD(...)
#define LOGE(...)
#endif // __ANDROID__
//TODO: infer batch size from model/use dynamic batch size
const unsigned int BATCH_SIZE = 1;
constexpr unsigned int BATCH_SIZE = 1;
//TODO: use dynamic sample rate
const unsigned int SAMPLE_RATE = 16000;
constexpr unsigned int SAMPLE_RATE = 16000;
const float AUDIO_WIN_LEN = 0.025f;
const float AUDIO_WIN_STEP = 0.01f;
const unsigned int AUDIO_WIN_LEN_SAMPLES = (unsigned int)(AUDIO_WIN_LEN * SAMPLE_RATE);
const unsigned int AUDIO_WIN_STEP_SAMPLES = (unsigned int)(AUDIO_WIN_STEP * SAMPLE_RATE);
constexpr float AUDIO_WIN_LEN = 0.032f;
constexpr float AUDIO_WIN_STEP = 0.02f;
constexpr unsigned int AUDIO_WIN_LEN_SAMPLES = (unsigned int)(AUDIO_WIN_LEN * SAMPLE_RATE);
constexpr unsigned int AUDIO_WIN_STEP_SAMPLES = (unsigned int)(AUDIO_WIN_STEP * SAMPLE_RATE);
const unsigned int MFCC_FEATURES = 26;
constexpr unsigned int MFCC_FEATURES = 26;
const float PREEMPHASIS_COEFF = 0.97f;
const unsigned int N_FFT = 512;
const unsigned int N_FILTERS = 26;
const unsigned int LOWFREQ = 0;
const unsigned int CEP_LIFTER = 22;
constexpr float PREEMPHASIS_COEFF = 0.97f;
constexpr unsigned int N_FFT = 512;
constexpr unsigned int N_FILTERS = 26;
constexpr unsigned int LOWFREQ = 0;
constexpr unsigned int CEP_LIFTER = 22;
using namespace tensorflow;
constexpr size_t WINDOW_SIZE = AUDIO_WIN_LEN * SAMPLE_RATE;
std::array<float, WINDOW_SIZE> calc_hamming_window() {
std::array<float, WINDOW_SIZE> a{0};
for (int i = 0; i < WINDOW_SIZE; ++i) {
a[i] = 0.54 - 0.46 * std::cos(2*M_PI*i/(WINDOW_SIZE-1));
}
return a;
}
std::array<float, WINDOW_SIZE> hamming_window = calc_hamming_window();
#ifndef USE_TFLITE
using namespace tensorflow;
#else
using namespace tflite;
#endif
using std::vector;
@ -77,7 +115,6 @@ struct StreamingState {
float last_sample; // used for preemphasis
vector<float> mfcc_buffer;
vector<float> batch_buffer;
bool skip_next_mfcc;
ModelState* model;
void feedAudioContent(const short* buffer, unsigned int buffer_size);
@ -92,9 +129,14 @@ struct StreamingState {
};
struct ModelState {
#ifndef USE_TFLITE
MemmappedEnv* mmap_env;
Session* session;
GraphDef graph_def;
#else // USE_TFLITE
std::unique_ptr<Interpreter> interpreter;
std::unique_ptr<FlatBufferModel> fbmodel;
#endif // USE_TFLITE
unsigned int ncep;
unsigned int ncontext;
Alphabet* alphabet;
@ -104,6 +146,12 @@ struct ModelState {
unsigned int mfcc_feats_per_timestep;
unsigned int n_context;
#ifdef USE_TFLITE
size_t previous_state_size;
std::unique_ptr<float[]> previous_state_c_;
std::unique_ptr<float[]> previous_state_h_;
#endif
ModelState();
~ModelState();
@ -132,8 +180,14 @@ struct ModelState {
};
ModelState::ModelState()
: mmap_env(nullptr)
:
#ifndef USE_TFLITE
mmap_env(nullptr)
, session(nullptr)
#else // USE_TFLITE
interpreter(nullptr)
, fbmodel(nullptr)
#endif // USE_TFLITE
, ncep(0)
, ncontext(0)
, alphabet(nullptr)
@ -142,20 +196,27 @@ ModelState::ModelState()
, n_steps(-1)
, mfcc_feats_per_timestep(-1)
, n_context(-1)
#ifdef USE_TFLITE
, previous_state_size(0)
, previous_state_c_(nullptr)
, previous_state_h_(nullptr)
#endif
{
}
ModelState::~ModelState()
{
#ifndef USE_TFLITE
if (session) {
Status status = session->Close();
if (!status.ok()) {
std::cerr << "Error closing TensorFlow session: " << status << std::endl;
}
}
delete mmap_env;
#endif // USE_TFLITE
delete scorer;
delete mmap_env;
delete alphabet;
}
@ -214,16 +275,11 @@ StreamingState::finishStream()
void
StreamingState::processAudioWindow(const vector<float>& buf)
{
skip_next_mfcc = !skip_next_mfcc;
if (!skip_next_mfcc) { // Was true
return;
}
// Compute MFCC features
float* mfcc;
int n_frames = csf_mfcc(buf.data(), buf.size(), SAMPLE_RATE,
AUDIO_WIN_LEN, AUDIO_WIN_STEP, MFCC_FEATURES, N_FILTERS, N_FFT,
LOWFREQ, SAMPLE_RATE/2, 0.f, CEP_LIFTER, 1, nullptr,
LOWFREQ, SAMPLE_RATE/2, 0.f, CEP_LIFTER, 1, hamming_window.data(),
&mfcc);
assert(n_frames == 1);
@ -286,6 +342,7 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector<float>& logi
{
const size_t num_classes = alphabet->GetSize() + 1; // +1 for blank
#ifndef USE_TFLITE
Tensor input(DT_FLOAT, TensorShape({BATCH_SIZE, n_steps, 2*n_context+1, MFCC_FEATURES}));
auto input_mapped = input.flat<float>();
@ -315,6 +372,41 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector<float>& logi
for (int t = 0; t < n_frames * BATCH_SIZE * num_classes; ++t) {
logits_output.push_back(logits_mapped(t));
}
#else // USE_TFLITE
// Feeding input_node
float* input_node = interpreter->typed_tensor<float>(interpreter->inputs()[0]);
{
int i;
for (i = 0; i < n_frames*mfcc_feats_per_timestep; ++i) {
input_node[i] = aMfcc[i];
}
for (; i < n_steps*mfcc_feats_per_timestep; ++i) {
input_node[i] = 0;
}
}
assert(previous_state_size > 0);
// Feeding previous_state_c, previous_state_h
memcpy(interpreter->typed_tensor<float>(interpreter->inputs()[1]), previous_state_c_.get(), sizeof(float) * previous_state_size);
memcpy(interpreter->typed_tensor<float>(interpreter->inputs()[2]), previous_state_h_.get(), sizeof(float) * previous_state_size);
TfLiteStatus status = interpreter->Invoke();
if (status != kTfLiteOk) {
std::cerr << "Error running session: " << status << "\n";
return;
}
float* outputs = interpreter->typed_tensor<float>(interpreter->outputs()[0]);
// The CTCDecoder works with log-probs.
for (int t = 0; t < n_frames * BATCH_SIZE * num_classes; ++t) {
logits_output.push_back(outputs[t]);
}
memcpy(previous_state_c_.get(), interpreter->typed_tensor<float>(interpreter->outputs()[1]), sizeof(float) * previous_state_size);
memcpy(previous_state_h_.get(), interpreter->typed_tensor<float>(interpreter->outputs()[2]), sizeof(float) * previous_state_size);
#endif // USE_TFLITE
}
char*
@ -345,7 +437,9 @@ DS_CreateModel(const char* aModelPath,
ModelState** retval)
{
std::unique_ptr<ModelState> model(new ModelState());
#ifndef USE_TFLITE
model->mmap_env = new MemmappedEnv(Env::Default());
#endif // USE_TFLITE
model->ncep = aNCep;
model->ncontext = aNContext;
model->alphabet = new Alphabet(aAlphabetConfigPath);
@ -357,9 +451,14 @@ DS_CreateModel(const char* aModelPath,
if (!aModelPath || strlen(aModelPath) < 1) {
std::cerr << "No model specified, cannot continue." << std::endl;
#ifndef USE_TFLITE
return error::INVALID_ARGUMENT;
#else // USE_TFLITE
return EINVAL;
#endif // USE_TFLITE
}
#ifndef USE_TFLITE
Status status;
SessionOptions options;
@ -441,6 +540,62 @@ DS_CreateModel(const char* aModelPath,
*retval = model.release();
return tensorflow::error::OK;
#else // USE_TFLITE
TfLiteStatus status;
model->fbmodel = tflite::FlatBufferModel::BuildFromFile(aModelPath);
if (status != kTfLiteOk) {
std::cerr << status << std::endl;
return status;
}
tflite::ops::builtin::BuiltinOpResolver resolver;
status = tflite::InterpreterBuilder(*model->fbmodel, resolver)(&model->interpreter);
if (status != kTfLiteOk) {
std::cerr << status << std::endl;
return status;
}
model->interpreter->AllocateTensors();
model->interpreter->SetNumThreads(4);
TfLiteIntArray* dims_input_node = model->interpreter->tensor(model->interpreter->inputs()[0])->dims;
model->n_steps = dims_input_node->data[1];
model->n_context = (dims_input_node->data[2] - 1 ) / 2;
model->mfcc_feats_per_timestep = dims_input_node->data[2] * dims_input_node->data[3];
TfLiteIntArray* dims_logits = model->interpreter->tensor(model->interpreter->outputs()[0])->dims;
const int final_dim_size = dims_logits->data[1] - 1;
if (final_dim_size != model->alphabet->GetSize()) {
std::cerr << "Error: Alphabet size does not match loaded model: alphabet "
<< "has size " << model->alphabet->GetSize()
<< ", but model has " << final_dim_size
<< " classes in its output. Make sure you're passing an alphabet "
<< "file with the same size as the one used for training."
<< std::endl;
return EINVAL;
}
const int previous_state_c_id = model->interpreter->inputs()[1];
const int previous_state_h_id = model->interpreter->inputs()[2];
TfLiteIntArray* dims_c = model->interpreter->tensor(previous_state_c_id)->dims;
TfLiteIntArray* dims_h = model->interpreter->tensor(previous_state_h_id)->dims;
assert(dims_c->data[1] == dims_h->data[1]);
model->previous_state_size = dims_c->data[1];
model->previous_state_c_.reset(new float[model->previous_state_size]());
model->previous_state_h_.reset(new float[model->previous_state_size]());
// Set initial values for previous_state_c and previous_state_h
memset(model->previous_state_c_.get(), 0, sizeof(float) * model->previous_state_size);
memset(model->previous_state_h_.get(), 0, sizeof(float) * model->previous_state_size);
*retval = model.release();
return kTfLiteOk;
#endif // USE_TFLITE
}
void
@ -476,7 +631,11 @@ DS_SpeechToText(ModelState* aCtx,
{
StreamingState* ctx;
int status = DS_SetupStream(aCtx, 0, aSampleRate, &ctx);
#ifndef USE_TFLITE
if (status != tensorflow::error::OK) {
#else // USE_TFLITE
if (status != kTfLiteOk) {
#endif // USE_TFLITE
return nullptr;
}
DS_FeedAudioContent(ctx, aBuffer, aBufferSize);
@ -491,16 +650,22 @@ DS_SetupStream(ModelState* aCtx,
{
*retval = nullptr;
#ifndef USE_TFLITE
Status status = aCtx->session->Run({}, {}, {"initialize_state"}, nullptr);
if (!status.ok()) {
std::cerr << "Error running session: " << status << std::endl;
return status.code();
}
#endif // USE_TFLITE
std::unique_ptr<StreamingState> ctx(new StreamingState());
if (!ctx) {
std::cerr << "Could not allocate streaming state." << std::endl;
#ifndef USE_TFLITE
return status.code();
#else // USE_TFLITE
return ENOMEM;
#endif // USE_TFLITE
}
const size_t num_classes = aCtx->alphabet->GetSize() + 1; // +1 for blank
@ -518,12 +683,14 @@ DS_SetupStream(ModelState* aCtx,
ctx->mfcc_buffer.resize(MFCC_FEATURES*aCtx->n_context, 0.f);
ctx->batch_buffer.reserve(aCtx->n_steps * aCtx->mfcc_feats_per_timestep);
ctx->skip_next_mfcc = false;
ctx->model = aCtx;
*retval = ctx.release();
#ifndef USE_TFLITE
return tensorflow::error::OK;
#else // USE_TFLITE
return kTfLiteOk;
#endif // USE_TFLITE
}
void
@ -624,7 +791,12 @@ DS_AudioToInputVector(const short* aBuffer,
void
DS_PrintVersions() {
#ifndef __ANDROID__
std::cerr << "TensorFlow: " << tf_git_version() << std::endl;
std::cerr << "DeepSpeech: " << ds_git_version() << std::endl;
#else
LOGE("DeepSpeech: %s", ds_git_version());
LOGD("DeepSpeech: %s", ds_git_version());
#endif
}