mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1539 from tilmankamp/remove-old-scripts

Browse Source 
Removing old training scripts and website support - #1534 #1538
This commit is contained in:
Tilman Kamp 2018-09-13 16:53:57 +02:00 committed by GitHub
commit 4e87886dd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 0 additions and 740 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
bin/enqueue.sh
View File

@ -1,118 +0,0 @@
#!/bin/bash
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi
GLOBAL_LOG="\\/dev\\/null" # by default there is no global logging
NODES=2 # to be adjusted
GPUS=8 # to be adjusted
HASH=`git rev-parse HEAD`
TO_RUN=run-`date +%Y-%m-%d-%H-%M` # default run name is based on user's name and current date time
runs_dir="/data/runs"
# parsing parameters
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
-h|--help)
echo "Usage: enqueue.sh [--help] [-g|--gpus N] [-n|--nodes N] [-l|-log file] [-c|--continue run] [run]"
echo ""
echo "--help print this help message"
echo "--gpus N allocates N gpus per node"
echo "--nodes N allocates N nodes"
echo "--log file if set, global logs will get appended to the provided file"
echo "--continue run continue a former run by copying its transitional content into the new run directory"
echo "run name of or path to a run directory - if just a name, it will be created at $runs_dir/<run>"
echo " defaults to user's name plus current date/time"
echo
exit 0
;;
-g|--gpus)
GPUS="$2"
shift
;;
-n|--nodes)
NODES="$2"
shift
;;
-l|--log)
GLOBAL_LOG=$(sed 's/[\/&]/\\&/g' <<< $2) # path-slashes are to be escaped
shift
;;
-c|--continue)
TO_CONTINUE="$2"
shift
;;
*)
TO_RUN="$1"
;;
esac
shift # past argument or value
done
# creates a run directory path from a run name given by $1
# if run name is a directory, it's treated relative to current directory
# if just a name, it will be treated relative to $runs_dir
function to_run_dir {
local run_dir
if [[ $1 == .*\/.* ]]; then
if [[ $1 == \/.* ]]; then
run_dir=$1
else
run_dir=$runs_dir/$1
fi
else
run_dir=$runs_dir/$1
fi
echo ${run_dir//\/\//\/}
}
# run directory from which "keep" data is to be copied over into the new run directory
if [ "$TO_CONTINUE" ]; then
CONTINUE_DIR=$(to_run_dir "$TO_CONTINUE")
if [ ! -d "$CONTINUE_DIR/results/keep" ]; then
echo "Cannot continue former run. Directory $CONTINUE_DIR/results/keep doesn't exist."
exit 1
fi
fi
# preparing run directory
RUN_DIR=$(to_run_dir "$TO_RUN")
echo "Creating run directory $RUN_DIR..."
if [ -d "$RUN_DIR" ]; then
echo "Run directory $RUN_DIR already exists."
exit 1
elif [ -f "$RUN_DIR" ]; then
echo "Run directory path $RUN_DIR is already a file."
exit 1
fi
mkdir -p "$RUN_DIR/src"
mkdir -p "$RUN_DIR/results"
# copying local tree into "src" sub-directory of run directory
# excluding .git
rsync -av . "$RUN_DIR/src" --exclude=/.git
# copying over "keep" data from continue run directory
if [ "$CONTINUE_DIR" ]; then
cp -rf "$CONTINUE_DIR/results/keep" "$RUN_DIR/results/"
fi
mkdir -p "$RUN_DIR/results/keep"
# patch-creating job.sbatch file from job-template with our parameters
sed \
-e "s/__ID__/$HASH/g" \
-e "s/__NAME__/$TO_RUN/g" \
-e "s/__GLOBAL_LOG__/$GLOBAL_LOG/g" \
-e "s/__NODES__/$NODES/g" \
-e "s/__GPUS__/$GPUS/g" \
bin/job-template.sbatch > $RUN_DIR/job.sbatch
# enqueuing the new job description and run directory
sbatch -D $RUN_DIR $RUN_DIR/job.sbatch

38
bin/run-fisher.sh
View File

@ -1,38 +0,0 @@
#!/bin/sh
set -xe
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi;
if [ ! -d "${COMPUTE_DATA_DIR}" ]; then
COMPUTE_DATA_DIR="data"
fi;
# Warn if we can't find the train files
if [ ! -f "${COMPUTE_DATA_DIR}/fisher-train.csv" ]; then
echo "Warning: It looks like you don't have the Fisher corpus" \
"downloaded and preprocessed. Make sure \$COMPUTE_DATA_DIR points to the" \
"folder where the Fisher data is located, and that you ran the" \
"importer script at bin/import_fisher.py before running this script."
fi;
if [ -d "${COMPUTE_KEEP_DIR}" ]; then
checkpoint_dir=$COMPUTE_KEEP_DIR
else
checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/fisher"))')
fi
python -u DeepSpeech.py \
--train_files "$COMPUTE_DATA_DIR/fisher-train.csv" \
--dev_files "$COMPUTE_DATA_DIR/fisher-dev.csv" \
--test_files "$COMPUTE_DATA_DIR/fisher-test.csv" \
--train_batch_size 32 \
--dev_batch_size 32 \
--test_batch_size 32 \
--learning_rate 0.0001 \
--epoch=20 \
--display_step 1 \
--validation_step 5 \
--checkpoint_dir "$checkpoint_dir" \
"$@"

40
bin/run-librivox.sh
View File

@ -1,40 +0,0 @@
#!/bin/sh
set -xe
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi;
if [ ! -d "${COMPUTE_DATA_DIR}" ]; then
COMPUTE_DATA_DIR="data"
fi;
# Warn if we can't find the train files
if [ ! -f "${COMPUTE_DATA_DIR}/librivox-train-clean-100.csv" ]; then
echo "Warning: It looks like you don't have the LibriSpeech corpus" \
"downloaded and preprocessed. Make sure \$COMPUTE_DATA_DIR points to the" \
"folder where the LibriSpeech data is located, and that you ran the" \
"importer script at bin/import_librivox.py before running this script."
fi;
if [ -d "${COMPUTE_KEEP_DIR}" ]; then
checkpoint_dir=$COMPUTE_KEEP_DIR
else
checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/librivox"))')
fi
python -u DeepSpeech.py \
--train_files "$COMPUTE_DATA_DIR/librivox-train-clean-100.csv,$COMPUTE_DATA_DIR/librivox-train-clean-360.csv,$COMPUTE_DATA_DIR/librivox-train-other-500.csv" \
--dev_files "$COMPUTE_DATA_DIR/librivox-dev-clean.csv,$COMPUTE_DATA_DIR/librivox-dev-other.csv" \
--test_files "$COMPUTE_DATA_DIR/librivox-test-clean.csv,$COMPUTE_DATA_DIR/librivox-test-other.csv" \
--train_batch_size 12 \
--dev_batch_size 12 \
--test_batch_size 12 \
--learning_rate 0.0001 \
--epoch 15 \
--display_step 5 \
--validation_step 5 \
--dropout_rate 0.30 \
--default_stddev 0.046875 \
--checkpoint_dir "$checkpoint_dir" \
"$@"

42
bin/run-swb.sh
View File

@ -1,42 +0,0 @@
#!/bin/sh
set -xe
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi;
if [ ! -d "${COMPUTE_DATA_DIR}" ]; then
COMPUTE_DATA_DIR="data"
fi;
# Warn if we can't find the train files
if [ ! -f "${COMPUTE_DATA_DIR}/swb-train.csv" ]; then
echo "Warning: It looks like you don't have the Switchboard corpus" \
"downloaded and preprocessed. Make sure \$COMPUTE_DATA_DIR points to the" \
"folder where the Switchboard data is located, and that you ran the" \
"importer script at bin/import_swb.py before running this script."
fi;
if [ -d "${COMPUTE_KEEP_DIR}" ]; then
checkpoint_dir=$COMPUTE_KEEP_DIR
else
checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/swb"))')
fi
python -u DeepSpeech.py \
--train_files "${COMPUTE_DATA_DIR}/LDC/LDC97S62/swb-train.csv" \
--dev_files "${COMPUTE_DATA_DIR}/LDC/LDC97S62/swb-dev.csv" \
--test_files "${COMPUTE_DATA_DIR}/LDC/LDC97S62/swb-test.csv" \
--train_batch_size 13 \
--dev_batch_size 13 \
--test_batch_size 13 \
--epoch 15 \
--learning_rate 0.0001 \
--display_step 0 \
--validation_step 1 \
--dropout_rate 0.15 \
--default_stddev 0.046875 \
--checkpoint_step 1 \
--checkpoint_dir "${COMPUTE_KEEP_DIR}" \
--wer_log_pattern "GLOBAL LOG: logwer('${COMPUTE_ID}', '%s', '%s', %f)"\
"$@"

40
bin/run-ted.sh
View File

@ -1,40 +0,0 @@
#!/bin/sh
set -xe
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi;
if [ ! -d "${COMPUTE_DATA_DIR}" ]; then
COMPUTE_DATA_DIR="data"
fi;
# Warn if we can't find the train files
if [ ! -f "${COMPUTE_DATA_DIR}/ted-train.csv" ]; then
echo "Warning: It looks like you don't have the TED-LIUM corpus downloaded"\
"and preprocessed. Make sure \$COMPUTE_DATA_DIR points to the folder where"\
"folder where the TED-LIUM data is located, and that you ran the" \
"importer script at bin/import_ted.py before running this script."
fi;
if [ -d "${COMPUTE_KEEP_DIR}" ]; then
checkpoint_dir=$COMPUTE_KEEP_DIR
else
checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/ted"))')
fi
python -u DeepSpeech.py \
--train_files "$COMPUTE_DATA_DIR/ted-train.csv" \
--dev_files "$COMPUTE_DATA_DIR/ted-dev.csv" \
--test_files "$COMPUTE_DATA_DIR/ted-test.csv" \
--train_batch_size 16 \
--dev_batch_size 8 \
--test_batch_size 8 \
--epoch 10 \
--display_step 10 \
--validation_step 1 \
--dropout_rate 0.30 \
--default_stddev 0.046875 \
--learning_rate 0.0001 \
--checkpoint_dir "$checkpoint_dir" \
"$@"

29
bin/run-timit.sh
View File

@ -1,29 +0,0 @@
#!/bin/sh
set -xe
if [ ! -f DeepSpeech.py ]; then
echo "Please make sure you run this from DeepSpeech's top level directory."
exit 1
fi;
if [ ! -f "data/TIMIT/timit_train.csv" ]; then
echo "Trying to preprocess TIMIT data, saving in ./data/TIMIT"
python -u bin/import_timit.py ./data/
fi;
if [ -d "${COMPUTE_KEEP_DIR}" ]; then
checkpoint_dir=$COMPUTE_KEEP_DIR
else
checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/timit"))')
fi
python -u DeepSpeech.py \
--train_files data/TIMIT/timit_train.csv \
--dev_files data/TIMIT/timit_test.csv \
--test_files data/TIMIT/timit_test.csv \
--train_batch_size 8 \
--dev_batch_size 8 \
--test_batch_size 8 \
--n_hidden 494 \
--epoch 50 \
--checkpoint_dir "$checkpoint_dir" \
"$@"

9
bin/update-website.sh
View File

@ -1,9 +0,0 @@
#!/bin/sh
#
# Trvial tool to warp manual update of website
#
set -xe
python util/website.py

188
index.htm
View File

@ -1,188 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>DeepSpeech reports</title>
<link rel="stylesheet" href="resources/bootstrap.min.css" />
<link rel="stylesheet" href="resources/jquery-ui.min.css" />
<link rel="stylesheet" href="resources/rickshaw.min.css" />
<script src="resources/jquery-3.1.1.min.js"></script>
<script src="resources/jquery-ui.min.js"></script>
<script src="resources/d3.v3.min.js"></script>
<script src="resources/rickshaw.min.js"></script>
<script type="text/javascript">
window.ALL_THE_DATA = [];
window.logwer = function(hash, time, type, wer) {
window.ALL_THE_DATA.push({ hash: hash, time: time, type: type, wer: wer });
};
</script>
<script src="werlog.js"></script>
<style type="text/css">
.rickshaw_legend {
color: black;
background: transparent;
}
.rickshaw_legend .line {
display: inline;
}
</style>
</head>
<body>
<div id="panels" class="container"></div>
<script src="resources/bootstrap.min.js"></script>
<script type="text/javascript">
const createChart = (id, title, fineprint, series, times) => {
var panelRoot = $('#panels');
var panel = $('<div/>').attr('id', id); panelRoot.append(panel);
panel.append('<h1>' + title + '</h1>');
panel.append('<p>' + fineprint + '</p>');
var cContainer = $('<div/>'); panel.append(cContainer);
var cChart = $('<div/>'); cContainer.append(cChart);
var cPreview = $('<div/>'); cContainer.append(cPreview);
var cTimeline = $('<div/>'); cContainer.append(cTimeline);
var cLegend = $('<div/>'); cContainer.append(cLegend);
var graph = new Rickshaw.Graph({
element: cChart[0],
height: 500,
renderer: 'area',
interpolation: 'linear',
unstack: 'true',
stroke: true,
preserve: true,
series: series
});
graph.render();
var preview = new Rickshaw.Graph.RangeSlider({
graph: graph,
element: cPreview[0]
});
var hoverDetail = new Rickshaw.Graph.HoverDetail({
graph: graph,
xFormatter: function(x) {
if (times)
return new Date(x * 1000.0).toString();
else
return x;
},
yFormatter: function(y) {
return (y * 100).toFixed(2) + '%';
}
});
var legend = new Rickshaw.Graph.Legend({
graph: graph,
element: cLegend[0]
});
var shelving = new Rickshaw.Graph.Behavior.Series.Toggle({
graph: graph,
legend: legend
});
var order = new Rickshaw.Graph.Behavior.Series.Order({
graph: graph,
legend: legend
});
var highlighter = new Rickshaw.Graph.Behavior.Series.Highlight({
graph: graph,
legend: legend
});
var ticksTreatment = 'glow';
if (times) {
var xAxis = new Rickshaw.Graph.Axis.Time({
graph: graph,
ticksTreatment: ticksTreatment,
timeFixture: new Rickshaw.Fixtures.Time.Local()
});
var annotator = new Rickshaw.Graph.Annotate({
graph: graph,
element: cTimeline[0]
});
times.forEach(time => annotator.add(time, new Date(time * 1000.0).toString()));
annotator.update();
xAxis.render();
}
var yAxis = new Rickshaw.Graph.Axis.Y({
graph: graph,
tickFormat: function(y) { return (y * 100).toFixed(0) + '%'; },
ticksTreatment: ticksTreatment
});
yAxis.render();
};
var ds = window.ALL_THE_DATA,
i,
reports = {},
overall_test_wer = [],
overall_validation_wer = [],
overall_train_wer = [],
times = [];
ds.forEach(item => {
var time = new Date(item.time).getTime() / 1000.0;
if (item.type == 'train')
overall_train_wer.push({ x: time, y: item.wer });
else if (item.type == 'dev')
overall_validation_wer.push({ x: time, y: item.wer });
else
overall_test_wer.push({ x: time, y: item.wer });
times.push(time);
});
const stdtext =
'The Y axis plots the mean word error rate ' +
'<a href="https://en.wikipedia.org/wiki/Word_error_rate">WER</a> ' +
'(of the training\'s last epoch).' +
'<p>Lower is better: 0% WER equates to "all words were understood correctly", ' +
'while 100% WER equates to "every single word was wrong". ' +
'Percentages greater than 100% may occur, if the "understood" phrase got longer than the original one.</p><p>' +
'<strong>Train WER</strong>, <strong>Train WER</strong> and <strong>Train WER</strong> ' +
'refer to the WERs calculated on base of the corresponding ' +
'<a href="https://en.wikipedia.org/wiki/Test_set#Validation_set">data-sets.</a></p><p>' +
'Please use the legend for turning on/off data series.</p>';
var groups = [];
if (overall_validation_wer.length > 0)
groups.push({ color: 'rgba(100,100,255,0.5)', data: overall_validation_wer, name: 'Validate WER' });
if (overall_train_wer.length > 0)
groups.push({ color: 'rgba(255,100,100,0.5)', data: overall_train_wer, name: 'Train WER' });
if (overall_test_wer.length > 0)
groups.push({ color: 'rgba(100,255,100,0.5)', data: overall_test_wer, name: 'Test WER' });
createChart(
'overall_wer',
'DeepSpeech Word Error Rate over time',
'The X axis plots all trainings by their start times. ' + stdtext,
groups,
times
);
</script>
</body>
</html>

6
resources/bootstrap.min.css vendored
View File

File diff suppressed because one or more lines are too long

7
resources/bootstrap.min.js vendored
View File

File diff suppressed because one or more lines are too long

5
resources/d3.v3.min.js vendored
View File

File diff suppressed because one or more lines are too long

4
resources/jquery-3.1.1.min.js vendored
View File

File diff suppressed because one or more lines are too long

7
resources/jquery-ui.min.css vendored
View File

File diff suppressed because one or more lines are too long

13
resources/jquery-ui.min.js vendored
View File

File diff suppressed because one or more lines are too long

1
resources/rickshaw.min.css vendored
View File

File diff suppressed because one or more lines are too long

3
resources/rickshaw.min.js vendored
View File

File diff suppressed because one or more lines are too long

190
util/website.py
View File

@ -1,190 +0,0 @@
from __future__ import print_function
from __future__ import absolute_import
import os
import paramiko
import pysftp
import sys
from bs4 import BeautifulSoup
def parse_for_deps(filename):
"""
This takes an HTML file as input and output a list of existing depenencies.
Empty output means something was wrong.
Dependency is one of: js script loaded, css stylesheet loaded
"""
with open(filename, 'r') as code:
soup = BeautifulSoup(code.read(), 'html.parser')
external_links = filter(lambda x: x is not None, [ link.get('href') for link in soup.find_all('link') ])
external_scripts = filter(lambda x: x is not None, [ script.get('src') for script in soup.find_all('script') ])
# Verify with stat()
try:
all_resources = map(lambda x: os.stat(x), external_links + external_scripts)
except OSError as ex:
if ex.errno == 2:
print("Missing dependency", ex)
return []
raise ex
# Try to read the file
try:
readable_resources = map(lambda x: os.open(x, os.O_RDONLY), external_links + external_scripts)
map(lambda x: os.close(x), readable_resources)
except OSError as ex:
print("Unreadable dependency", ex)
return []
# It should be all good.
return external_links + external_scripts
def get_ssh(auth_infos):
"""
Connect to SSH server
"""
cinfo = {'host': auth_infos['ds_website_server_fqdn'],
'username': auth_infos['ds_website_username'],
'private_key': auth_infos['ds_website_privkey'],
'port': auth_infos['ds_website_server_port']}
return pysftp.Connection(**cinfo)
def verify_ssh_dir(auth_infos):
"""
This should ensure that SSH connection works and that the directory where
we want to push data is either empty or contains at least a .htaccess and
index.htm file
"""
print("Checking connection: %s@%s:%s (port %d)" % (auth_infos['ds_website_username'], auth_infos['ds_website_server_fqdn'], auth_infos['ds_website_server_root'], auth_infos['ds_website_server_port']))
try:
with get_ssh(auth_infos) as sftp:
with sftp.cd(auth_infos['ds_website_server_root']):
files = sftp.listdir()
if len(files) == 0 or (('.htaccess' in files) and ('index.htm' in files)):
return True
else:
print("Invalid content for %s:" % (auth_infos['ds_website_server_root']), files)
return False
except paramiko.ssh_exception.AuthenticationException as ex:
print("Authentication error, please verify credentials")
return False
except IOError as ex:
print("Unable to read a file (private key or invalid path on server ?)", ex)
return False
# Should not be reached
return False
def push_files_sftp(files, auth_infos):
"""
This will push all of the ``files`` listed to the server root folder.
"""
# Directories that we might be required to create
dirs = filter(lambda x: len(x) > 0, list(set([ os.path.dirname(x) for x in files ])))
created = []
pushed = []
try:
with get_ssh(auth_infos) as sftp:
with sftp.cd(auth_infos['ds_website_server_root']):
# Create dirs if needed, they all should depend from the root
for dir in dirs:
if not sftp.isdir(dir):
print("Creating directory", dir)
sftp.makedirs(dir)
created.append(dir)
# Push all files, chdir() for each
for fil in files:
with sftp.cd(os.path.dirname(fil)):
print("Pushing", fil)
sftp.put(fil)
pushed.append(fil)
return pushed
except paramiko.ssh_exception.AuthenticationException as ex:
print("Authentication error, please verify credentials")
return False
except IOError as ex:
print("Unable to read a file (private key or invalid path on server ?)", ex)
return False
# Should not be reached
return False
def maybe_publish(file='index.htm'):
"""
Publishing to the web requires the following env variables to be set
'ds_website_username: defines the SSH username to connect to
'ds_website_privkey: defines the SSH privkey filename to use for connection
'ds_website_server_fqdn: hostname of the SSH server
'ds_website_server_port: port of the SSH server, defaults to 22
'ds_website_server_root: directory on the server where to push data,
should be either empty, or containing at least
a .htaccess and index.htm file
"""
ssh_auth_infos = {
'ds_website_username': '',
'ds_website_privkey': '',
'ds_website_server_fqdn': '',
'ds_website_server_port': 22,
'ds_website_server_root': ''
}
for key in ssh_auth_infos.keys():
vartype = type(ssh_auth_infos[key])
value = os.environ.get(key)
if value is not None:
if vartype == str:
ssh_auth_infos[key] = str(os.environ.get(key))
elif vartype == int:
try:
ssh_auth_infos[key] = int(os.environ.get(key))
except TypeError as ex:
print("WARNING:", "Keeping default SSH port value because of:", ex)
missing_env = [
x for x in ssh_auth_infos.keys()
if (len(str(ssh_auth_infos[x])) == 0)
]
if len(missing_env) > 0:
print("Not publishing, missing some required environment variables:", missing_env)
print("But maybe this is what you wanted, after all ...")
return False
all_deps = parse_for_deps(file)
if len(all_deps) == 0:
print("Problem during deps computation, aborting")
return False
if not verify_ssh_dir(ssh_auth_infos):
print("Problem during SSH directory verification, aborting")
return False
all_files = [ file ] + all_deps
uploaded = push_files_sftp(all_files, ssh_auth_infos)
if len(uploaded) == 0:
print("Unable to upload anything")
return False
elif len(uploaded) != len(all_files):
print("Partial upload has been completed:")
print("all_files=", all_files)
print("uploaded=", uploaded)
else:
print("Complete upload has been completed.")
return True
# Support CLI invocation
if __name__ == "__main__":
if maybe_publish():
print("All good!")
sys.exit(0)
else:
print("Error happened ...")
sys.exit(1)