From 6ff340f82ab564dd29410471cd8a875cea21c398 Mon Sep 17 00:00:00 2001 From: Kelly Davis Date: Thu, 27 Oct 2016 05:58:08 +0200 Subject: [PATCH 1/2] Fixes #101 --- util/importers/librivox.py | 37 ++++++++++++++++++++++++------------- util/importers/ted_lium.py | 37 ++++++++++++++++++++++++------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/util/importers/librivox.py b/util/importers/librivox.py index 8a1a4492..825391f6 100644 --- a/util/importers/librivox.py +++ b/util/importers/librivox.py @@ -73,25 +73,36 @@ class DataSet(object): def _populate_batch_queue(self): with self._graph.as_default(): - while True: - n_steps = 0 - sources = [] - targets = [] - for index, (txt_file, wav_file) in enumerate(self._files_circular_list): - if index >= self._batch_size: - break + n_steps = 0 + sources = [] + targets = [] + batch_index = 0 + for txt_file, wav_file in self._files_circular_list: + if batch_index < self._batch_size: next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) if n_steps < next_source.shape[0]: n_steps = next_source.shape[0] sources.append(next_source) with open(txt_file) as open_txt_file: targets.append(open_txt_file.read()) - target = texts_to_sparse_tensor(targets) - for index, next_source in enumerate(sources): - npad = ((0,(n_steps - next_source.shape[0])), (0,0)) - sources[index] = np.pad(next_source, pad_width=npad, mode='constant') - source = np.array(sources) - self._batch_queue.put((source, target)) + batch_index = batch_index + 1 + else: + # Put batch on queue + target = texts_to_sparse_tensor(targets) + for index, next_source in enumerate(sources): + npad = ((0,(n_steps - next_source.shape[0])), (0,0)) + sources[index] = np.pad(next_source, pad_width=npad, mode='constant') + source = np.array(sources) + self._batch_queue.put((source, target)) + # Deal with current txt_file, wav_file pair + next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) + n_steps = next_source.shape[0] + sources = [] + sources.append(next_source) + targets = [] + with open(txt_file) as open_txt_file: + targets.append(open_txt_file.read()) + batch_index = 1 def next_batch(self): source, target = self._batch_queue.get() diff --git a/util/importers/ted_lium.py b/util/importers/ted_lium.py index e48c45a5..5f673126 100644 --- a/util/importers/ted_lium.py +++ b/util/importers/ted_lium.py @@ -79,25 +79,36 @@ class DataSet(object): def _populate_batch_queue(self): with self._graph.as_default(): - while True: - n_steps = 0 - sources = [] - targets = [] - for index, (txt_file, wav_file) in enumerate(self._files_circular_list): - if index >= self._batch_size: - break + n_steps = 0 + sources = [] + targets = [] + batch_index = 0 + for txt_file, wav_file in self._files_circular_list: + if batch_index < self._batch_size: next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) if n_steps < next_source.shape[0]: n_steps = next_source.shape[0] sources.append(next_source) with open(txt_file) as open_txt_file: targets.append(open_txt_file.read()) - target = texts_to_sparse_tensor(targets) - for index, next_source in enumerate(sources): - npad = ((0,(n_steps - next_source.shape[0])), (0,0)) - sources[index] = np.pad(next_source, pad_width=npad, mode='constant') - source = np.array(sources) - self._batch_queue.put((source, target)) + batch_index = batch_index + 1 + else: + # Put batch on queue + target = texts_to_sparse_tensor(targets) + for index, next_source in enumerate(sources): + npad = ((0,(n_steps - next_source.shape[0])), (0,0)) + sources[index] = np.pad(next_source, pad_width=npad, mode='constant') + source = np.array(sources) + self._batch_queue.put((source, target)) + # Deal with current txt_file, wav_file pair + next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) + n_steps = next_source.shape[0] + sources = [] + sources.append(next_source) + targets = [] + with open(txt_file) as open_txt_file: + targets.append(open_txt_file.read()) + batch_index = 1 def next_batch(self): source, target = self._batch_queue.get() From 33e9912259a947804f3e109a26df5be88994a781 Mon Sep 17 00:00:00 2001 From: Kelly Davis Date: Thu, 27 Oct 2016 15:46:52 +0200 Subject: [PATCH 2/2] Updates suggested in first review of #102 --- util/importers/librivox.py | 26 ++++++++++---------------- util/importers/ted_lium.py | 26 ++++++++++---------------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/util/importers/librivox.py b/util/importers/librivox.py index 825391f6..9716fcfe 100644 --- a/util/importers/librivox.py +++ b/util/importers/librivox.py @@ -78,15 +78,7 @@ class DataSet(object): targets = [] batch_index = 0 for txt_file, wav_file in self._files_circular_list: - if batch_index < self._batch_size: - next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) - if n_steps < next_source.shape[0]: - n_steps = next_source.shape[0] - sources.append(next_source) - with open(txt_file) as open_txt_file: - targets.append(open_txt_file.read()) - batch_index = batch_index + 1 - else: + if batch_index == self._batch_size: # Put batch on queue target = texts_to_sparse_tensor(targets) for index, next_source in enumerate(sources): @@ -94,15 +86,17 @@ class DataSet(object): sources[index] = np.pad(next_source, pad_width=npad, mode='constant') source = np.array(sources) self._batch_queue.put((source, target)) - # Deal with current txt_file, wav_file pair - next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) - n_steps = next_source.shape[0] + n_steps = 0 sources = [] - sources.append(next_source) targets = [] - with open(txt_file) as open_txt_file: - targets.append(open_txt_file.read()) - batch_index = 1 + batch_index = 0 + next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) + if n_steps < next_source.shape[0]: + n_steps = next_source.shape[0] + sources.append(next_source) + with open(txt_file) as open_txt_file: + targets.append(open_txt_file.read()) + batch_index = batch_index + 1 def next_batch(self): source, target = self._batch_queue.get() diff --git a/util/importers/ted_lium.py b/util/importers/ted_lium.py index 5f673126..36dadf29 100644 --- a/util/importers/ted_lium.py +++ b/util/importers/ted_lium.py @@ -84,15 +84,7 @@ class DataSet(object): targets = [] batch_index = 0 for txt_file, wav_file in self._files_circular_list: - if batch_index < self._batch_size: - next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) - if n_steps < next_source.shape[0]: - n_steps = next_source.shape[0] - sources.append(next_source) - with open(txt_file) as open_txt_file: - targets.append(open_txt_file.read()) - batch_index = batch_index + 1 - else: + if batch_index == self._batch_size: # Put batch on queue target = texts_to_sparse_tensor(targets) for index, next_source in enumerate(sources): @@ -100,15 +92,17 @@ class DataSet(object): sources[index] = np.pad(next_source, pad_width=npad, mode='constant') source = np.array(sources) self._batch_queue.put((source, target)) - # Deal with current txt_file, wav_file pair - next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) - n_steps = next_source.shape[0] + n_steps = 0 sources = [] - sources.append(next_source) targets = [] - with open(txt_file) as open_txt_file: - targets.append(open_txt_file.read()) - batch_index = 1 + batch_index = 0 + next_source = audiofile_to_input_vector(wav_file, self._numcep, self._numcontext) + if n_steps < next_source.shape[0]: + n_steps = next_source.shape[0] + sources.append(next_source) + with open(txt_file) as open_txt_file: + targets.append(open_txt_file.read()) + batch_index = batch_index + 1 def next_batch(self): source, target = self._batch_queue.get()