using AsyncAwaitBestPractices.MVVM; using CSCore; using CSCore.CoreAudioAPI; using CSCore.SoundIn; using CSCore.Streams; using DeepSpeechClient.Interfaces; using DeepSpeechClient.Models; using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Win32; using System; using System.Collections.Concurrent; using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; namespace DeepSpeech.WPF.ViewModels { /// /// View model of the MainWindow View. /// public class MainWindowViewModel : BindableBase { #region Constants private const int SampleRate = 16000; private const string LMPath = "lm.binary"; private const string TriePath = "trie"; #endregion private readonly IDeepSpeech _sttClient; #region Commands /// /// Gets or sets the command that enables the language model. /// public IAsyncCommand EnableLanguageModelCommand { get; private set; } /// /// Gets or sets the command that runs inference using an audio file. /// public IAsyncCommand InferenceFromFileCommand { get; private set; } /// /// Gets or sets the command that opens a dialog to select an audio file. /// public RelayCommand SelectFileCommand { get; private set; } /// /// Gets or sets the command that starts to record. /// public RelayCommand StartRecordingCommand { get; private set; } /// /// Gets or sets the command that stops the recording and gets the result. /// public IAsyncCommand StopRecordingCommand { get; private set; } #endregion #region Streaming /// /// Stream used to feed data into the acoustic model. /// private DeepSpeechStream _sttStream; /// /// Records the audio of the selected device. /// private WasapiCapture _audioCapture; /// /// Converts the device source into a wavesource. /// private SoundInSource _soundInSource; /// /// Target wave source.(16KHz Mono 16bit for DeepSpeech) /// private IWaveSource _convertedSource; /// /// Queue that prevents feeding data to the inference engine if it is busy. /// private ConcurrentQueue _bufferQueue = new ConcurrentQueue(); private int _threadSafeBoolBackValue = 0; /// /// Lock to process items in the queue one at time. /// public bool StreamingIsBusy { get => (Interlocked.CompareExchange(ref _threadSafeBoolBackValue, 1, 1) == 1); set { if (value) Interlocked.CompareExchange(ref _threadSafeBoolBackValue, 1, 0); else Interlocked.CompareExchange(ref _threadSafeBoolBackValue, 0, 1); } } #endregion #region ViewProperties private bool _enableStartRecord; /// /// Gets or sets record status to control the record command. /// public bool EnableStartRecord { get => _enableStartRecord; set => SetProperty(ref _enableStartRecord, value); } private bool _stopRecordStopRecord; /// /// Gets or sets record status to control stop command. /// public bool EnableStopRecord { get => _stopRecordStopRecord; set => SetProperty(ref _stopRecordStopRecord, value, onChanged: ()=> ((AsyncCommand)StopRecordingCommand).RaiseCanExecuteChanged()); } private MMDevice _selectedDevice; /// /// Gets or sets the selected recording device. /// public MMDevice SelectedDevice { get => _selectedDevice; set => SetProperty(ref _selectedDevice, value, onChanged: UpdateSelectedDevice); } private string _statusMessage; /// /// Gets or sets status message. /// public string StatusMessage { get => _statusMessage; set => SetProperty(ref _statusMessage, value); } private bool _languageModelEnabled; /// /// Gets or sets the language model status. /// private bool LanguageModelEnabled { get => _languageModelEnabled; set => SetProperty(ref _languageModelEnabled, value, onChanged: () => ((AsyncCommand)EnableLanguageModelCommand).RaiseCanExecuteChanged()); } private bool _isRunningInference; /// /// Gets or sets whenever the model is running inference. /// private bool IsRunningInference { get => _isRunningInference; set => SetProperty(ref _isRunningInference, value, onChanged: () => ((AsyncCommand)InferenceFromFileCommand).RaiseCanExecuteChanged()); } private string _transcription; /// /// Gets or sets the current transcription. /// public string Transcription { get => _transcription; set => SetProperty(ref _transcription, value); } private string _audioFilePaht; /// /// Gets or sets the selected audio file path. /// public string AudioFilePath { get => _audioFilePaht; set => SetProperty(ref _audioFilePaht, value); } private ObservableCollection _deviceNames; /// /// Gets or sets the available recording devices. /// public ObservableCollection AvailableRecordDevices { get => _deviceNames; set => SetProperty(ref _deviceNames, value); } #endregion #region Ctors public MainWindowViewModel(IDeepSpeech sttClient) { _sttClient = sttClient; EnableLanguageModelCommand = new AsyncCommand(()=>EnableLanguageModelAsync(LMPath,TriePath), _ => !LanguageModelEnabled); InferenceFromFileCommand = new AsyncCommand(ExecuteInferenceFromFileAsync, _ => !IsRunningInference); SelectFileCommand = new RelayCommand(SelectAudioFile); StartRecordingCommand = new RelayCommand(StartRecording, canExecute: CanExecuteStartRecording); StopRecordingCommand = new AsyncCommand(StopRecordingAsync, _ => EnableStopRecord); LoadAvailableCaptureDevices(); } #endregion /// /// Releases the current capture device and initializes the selected one. /// private void UpdateSelectedDevice() { ReleaseCapture(); InitializeAudioCapture(); } /// /// Releases the capture device. /// private void ReleaseCapture() { if (_audioCapture != null) { _audioCapture.DataAvailable -= Capture_DataAvailable; _audioCapture.Dispose(); } } /// /// Command usage to know when the recording can start. /// /// If the device is not null. private bool CanExecuteStartRecording() => SelectedDevice != null; /// /// Loads all the available audio capture devices. /// private void LoadAvailableCaptureDevices() { AvailableRecordDevices = new ObservableCollection( MMDeviceEnumerator.EnumerateDevices(DataFlow.All, DeviceState.Active)); //we get only enabled devices EnableStartRecord = true; if (AvailableRecordDevices?.Count != 0) SelectedDevice = AvailableRecordDevices[0]; } /// /// Initializes the capture source. /// private void InitializeAudioCapture() { if (SelectedDevice != null) { _audioCapture = SelectedDevice.DataFlow == DataFlow.Capture ? new WasapiCapture() : new WasapiLoopbackCapture(); _audioCapture.Device = SelectedDevice; _audioCapture.Initialize(); _audioCapture.DataAvailable += Capture_DataAvailable; _soundInSource = new SoundInSource(_audioCapture) { FillWithZeros = false }; //create a source, that converts the data provided by the //soundInSource to required format _convertedSource = _soundInSource .ChangeSampleRate(SampleRate) // sample rate .ToSampleSource() .ToWaveSource(16); //bits per sample _convertedSource = _convertedSource.ToMono(); } } private void Capture_DataAvailable(object sender, DataAvailableEventArgs e) { //read data from the converedSource //important: don't use the e.Data here //the e.Data contains the raw data provided by the //soundInSource which won't have the deepspeech required audio format byte[] buffer = new byte[_convertedSource.WaveFormat.BytesPerSecond / 2]; int read; //keep reading as long as we still get some data while ((read = _convertedSource.Read(buffer, 0, buffer.Length)) > 0) { short[] sdata = new short[(int)Math.Ceiling(Convert.ToDecimal(read / 2))]; Buffer.BlockCopy(buffer, 0, sdata, 0, read); _bufferQueue.Enqueue(sdata); Task.Run(() => OnNewData()); } } /// /// Starts processing data from the queue. /// private void OnNewData() { while (!StreamingIsBusy && !_bufferQueue.IsEmpty) { if (_bufferQueue.TryDequeue(out short[] buffer)) { StreamingIsBusy = true; _sttClient.FeedAudioContent(_sttStream, buffer, Convert.ToUInt32(buffer.Length)); StreamingIsBusy = false; } } } /// /// Enables the language model. /// /// Language model path. /// Trie path. /// A Task to await. public async Task EnableLanguageModelAsync(string lmPath, string triePath) { try { StatusMessage = "Loading language model..."; const float LM_ALPHA = 0.75f; const float LM_BETA = 1.85f; await Task.Run(() => _sttClient.EnableDecoderWithLM(LMPath, TriePath, LM_ALPHA, LM_BETA)); LanguageModelEnabled = true; StatusMessage = "Language model loaded."; } catch (Exception ex) { StatusMessage = ex.Message; } } /// /// Runs inference and sets the transcription of an audio file. /// /// A Task to await. public async Task ExecuteInferenceFromFileAsync() { try { IsRunningInference = true; Transcription = string.Empty; StatusMessage = "Running inference..."; Stopwatch watch = new Stopwatch(); var waveBuffer = new NAudio.Wave.WaveBuffer(File.ReadAllBytes(AudioFilePath)); using (var waveInfo = new NAudio.Wave.WaveFileReader(AudioFilePath)) { watch.Start(); string speechResult = await Task.Run(() => _sttClient.SpeechToText( waveBuffer.ShortBuffer, Convert.ToUInt32(waveBuffer.MaxSize / 2))); watch.Stop(); Transcription = $"Audio duration: {waveInfo.TotalTime.ToString()} {Environment.NewLine}" + $"Inference took: {watch.Elapsed.ToString()} {Environment.NewLine}" + $"Recognized text: {speechResult}"; } waveBuffer.Clear(); StatusMessage = string.Empty; } catch (Exception ex) { StatusMessage = ex.Message; } finally { IsRunningInference = false; } } /// /// Stops the recording and sets the transcription of the closed stream. /// /// A Task to await. private async Task StopRecordingAsync() { EnableStopRecord = false; _audioCapture.Stop(); while (!_bufferQueue.IsEmpty && StreamingIsBusy) //we wait for all the queued buffers to be processed { await Task.Delay(90); } Transcription = _sttClient.FinishStream(_sttStream); EnableStartRecord = true; } /// /// Creates a new stream and starts the recording. /// private void StartRecording() { _sttStream =_sttClient.CreateStream(); _audioCapture.Start(); EnableStartRecord = false; EnableStopRecord = true; } /// /// Opens a dialog to select an audio file. /// private void SelectAudioFile() { OpenFileDialog dialog = new OpenFileDialog { Filter = "wav Files |*.wav", Multiselect = false, Title = "Please select a wav file." }; if ((bool)dialog.ShowDialog()) { AudioFilePath = dialog.FileName; } } } }