using DeepSpeechClient.Interfaces; using DeepSpeechClient.Structs; using System; using System.IO; using System.Runtime.InteropServices; using System.Text; namespace DeepSpeechClient { /// /// Client of the Mozilla's deepspeech implementation. /// public class DeepSpeech : IDeepSpeech, IDisposable { private unsafe ModelState** _modelStatePP; private unsafe ModelState* _modelStateP; private unsafe StreamingState** _streamingStatePP; public DeepSpeech() { } #region IDeepSpeech /// /// Create an object providing an interface to a trained DeepSpeech model. /// /// The path to the frozen model graph. /// The number of cepstrum the model was trained with. /// The context window the model was trained with. /// The path to the configuration file specifying the alphabet used by the network. /// The beam width used by the decoder. A larger beam width generates better results at the cost of decoding time. /// Zero on success, non-zero on failure. public unsafe int CreateModel(string aModelPath, uint aNCep, uint aNContext, string aAlphabetConfigPath, uint aBeamWidth) { string exceptionMessage = null; if (string.IsNullOrWhiteSpace(aModelPath)) { exceptionMessage = "Model path cannot be empty."; } if (string.IsNullOrWhiteSpace(aAlphabetConfigPath)) { exceptionMessage = "Alphabet path cannot be empty."; } if (!File.Exists(aModelPath)) { exceptionMessage = $"Cannot find the model file: {aModelPath}"; } if (!File.Exists(aAlphabetConfigPath)) { exceptionMessage = $"Cannot find the alphabet file: {aAlphabetConfigPath}"; } if (exceptionMessage != null) { throw new FileNotFoundException(exceptionMessage); } int result = NativeImp.DS_CreateModel(aModelPath, aNCep, aNContext, aAlphabetConfigPath, aBeamWidth, ref _modelStatePP); _modelStateP = *_modelStatePP; return result; } /// /// Frees associated resources and destroys models objects. /// public unsafe void Dispose() { NativeImp.DS_DestroyModel(_modelStatePP); } /// /// Enable decoding using beam scoring with a KenLM language model. /// /// The path to the configuration file specifying the alphabet used by the network. /// The path to the language model binary file. /// The path to the trie file build from the same vocabulary as the language model binary. /// The alpha hyperparameter of the CTC decoder. Language Model weight. /// The beta hyperparameter of the CTC decoder. Word insertion weight. /// Zero on success, non-zero on failure (invalid arguments). public unsafe int EnableDecoderWithLM(string aAlphabetConfigPath, string aLMPath, string aTriePath, float aLMAlpha, float aLMBeta) { string exceptionMessage = null; if (string.IsNullOrWhiteSpace(aTriePath)) { exceptionMessage = "Path to the trie file cannot be empty."; } if (!File.Exists(aTriePath)) { exceptionMessage = $"Cannot find the trie file: {aTriePath}"; } if (exceptionMessage != null) { throw new FileNotFoundException(exceptionMessage); } return NativeImp.DS_EnableDecoderWithLM(_modelStatePP, aAlphabetConfigPath, aLMPath, aTriePath, aLMAlpha, aLMBeta); } /// /// Feeds audio samples to an ongoing streaming inference. /// /// An array of 16-bit, mono raw audio samples at the appropriate sample rate. public unsafe void FeedAudioContent(short[] aBuffer, uint aBufferSize) { NativeImp.DS_FeedAudioContent(_streamingStatePP, aBuffer, aBufferSize); } /// /// Closes the ongoing streaming inference, returns the STT result over the whole audio signal. /// /// The STT result. The user is responsible for freeing the string. public unsafe string FinishStream() { return NativeImp.DS_FinishStream(_streamingStatePP); } /// /// Computes the intermediate decoding of an ongoing streaming inference. This is an expensive process as the decoder implementation isn't /// currently capable of streaming, so it always starts from the beginning of the audio. /// /// The STT intermediate result. The user is responsible for freeing the string. public unsafe string IntermediateDecode() { return NativeImp.DS_IntermediateDecode(_streamingStatePP); } /// /// Prints the versions of Tensorflow and DeepSpeech. /// public unsafe void PrintVersions() { NativeImp.DS_PrintVersions(); } /// /// Creates a new streaming inference state. /// /// Number of timestep frames to reserve. /// One timestep is equivalent to two window lengths(20ms). /// If set to 0 we reserve enough frames for 3 seconds of audio(150). /// The sample-rate of the audio signal /// Zero for success, non-zero on failure public unsafe int SetupStream(uint aPreAllocFrames, uint aSampleRate) { return NativeImp.DS_SetupStream(_modelStatePP, aPreAllocFrames, aSampleRate, ref _streamingStatePP); } /// /// Destroy a streaming state without decoding the computed logits. /// This can be used if you no longer need the result of an ongoing streaming /// inference and don't want to perform a costly decode operation. /// public unsafe void DiscardStream() { NativeImp.DS_DiscardStream(ref _streamingStatePP); } /// /// Use the DeepSpeech model to perform Speech-To-Text. /// /// A 16-bit, mono raw audio signal at the appropriate sample rate. /// The number of samples in the audio signal. /// The sample-rate of the audio signal. /// The STT result. The user is responsible for freeing the string. Returns NULL on error. public unsafe string SpeechToText(short[] aBuffer, uint aBufferSize, uint aSampleRate) { var res = NativeImp.DS_SpeechToText(_modelStatePP, aBuffer, aBufferSize, aSampleRate); int len = 0; while (Marshal.ReadByte(res, len) != 0) ++len; byte[] buffer = new byte[len]; Marshal.Copy(res, buffer, 0, buffer.Length); return Encoding.UTF8.GetString(buffer); } #endregion } }