mirrors/DeepSpeech
2
0
Fork 0
mirror of https://github.com/mozilla/DeepSpeech.git synced 2025-10-26 11:19:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Conversions to modern print statements

Browse Source
This commit is contained in:
Mike C. Fletcher 2017-03-26 14:58:51 -04:00
parent 5428880151
commit 288bd64a53
6 changed files with 67 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
util/automation.py
View File

@ -2,6 +2,7 @@
from __future__ import unicode_literals
from __future__ import print_function
import json
import os
import git
@ -65,21 +66,21 @@ class GPUUsage(Thread):
def stop(self):
if not self._process:
print "Trying to stop nvidia-smi but no more process, please fix."
print("Trying to stop nvidia-smi but no more process, please fix.")
return
print "Ending nvidia-smi monitoring: PID", self._process.pid
print("Ending nvidia-smi monitoring: PID", self._process.pid)
self._process.terminate()
print "Ended nvidia-smi monitoring ..."
print("Ended nvidia-smi monitoring ...")
def run(self):
print "Starting nvidia-smi monitoring"
print("Starting nvidia-smi monitoring")
# If the system has no CUDA setup, then this will fail.
try:
self._process = subprocess.Popen(self._cmd, stdout=subprocess.PIPE)
except OSError as ex:
print "Unable to start monitoring, check your environment:", ex
print("Unable to start monitoring, check your environment:", ex)
return
writer = None
@ -130,7 +131,7 @@ class GPUUsageChart():
for plot in self._rows:
self.produce_plot(plot)
except IOError as ex:
print "Unable to read", ex
print("Unable to read", ex)
def append_data(self, row):
for bucket, value in row.iteritems():
@ -147,18 +148,18 @@ class GPUUsageChart():
self._data[bucket][gpu] += [ value ]
def read(self):
print "Reading data from", self._csv
print("Reading data from", self._csv)
with open(self._csv, 'r') as f:
for r in csv.DictReader(f):
self.append_data(r)
def produce_plot(self, key, with_spline=True):
png = self._basename % (key, )
print "Producing plot for", key, "as", png
print("Producing plot for", key, "as", png)
fig, axis = plt.subplots()
data = self._data[key]
if data is None:
print "Data was empty, aborting"
print("Data was empty, aborting")
return
x = range(len(data[0]))
@ -339,10 +340,10 @@ def ensure_git_clone(sha):
# If non-existent, create a clone
if not os.path.isdir(gitdir):
source = get_github_repo_url()
print "Performing a fresh clone from", source, "to", DEEPSPEECH_CLONE_PATH
print("Performing a fresh clone from", source, "to", DEEPSPEECH_CLONE_PATH)
ds_repo = git.Repo.clone_from(source, DEEPSPEECH_CLONE_PATH)
else:
print "Using existing clone from", DEEPSPEECH_CLONE_PATH
print("Using existing clone from", DEEPSPEECH_CLONE_PATH)
ds_repo = git.Repo(gitdir)
# Ensure we have a valid non bare local clone
@ -412,7 +413,7 @@ def ensure_gpu_usage(root_dir):
gpu_usage_root = os.path.abspath(os.environ.get('ds_gpu_usage_root', root_dir))
gpu_usage_path = os.path.join(gpu_usage_root, get_git_desc())
print "Will produce CSV and charts in %s" % gpu_usage_path
print("Will produce CSV and charts in %s" % gpu_usage_path)
if not os.path.isdir(gpu_usage_path):
os.makedirs(gpu_usage_path)
@ -451,9 +452,9 @@ def ensure_checkpoint_directory():
# it does not exists.
# If it actually exists, the later os.path.isdir(checkpoint_dir) will
# take care of setting the checkpoint_restore value
print "Trying with maybe_checkpoint_dir", maybe_checkpoint_dir
print("Trying with maybe_checkpoint_dir", maybe_checkpoint_dir)
if not os.path.isdir(maybe_checkpoint_dir):
print "Trying to force-restore checkpoint from non-existent directory", maybe_checkpoint_dir
print("Trying to force-restore checkpoint from non-existent directory", maybe_checkpoint_dir)
else:
checkpoint_dir = maybe_checkpoint_dir
@ -461,10 +462,10 @@ def ensure_checkpoint_directory():
if not os.path.isdir(checkpoint_dir):
os.makedirs(checkpoint_dir)
else:
print "Found existing checkpoint dir, re-using it"
print("Found existing checkpoint dir, re-using it")
checkpoint_restore = True
print "Ensured checkpoint infos: ", checkpoint_dir, checkpoint_restore
print("Ensured checkpoint infos: ", checkpoint_dir, checkpoint_restore)
return checkpoint_dir, checkpoint_restore
@ -533,64 +534,64 @@ def exec_main():
if len(sha_to_run) == 0:
current = get_last_sha1()
if len(current) == 40:
print 'Existing SHA1:', current, 'fetching changes'
print('Existing SHA1:', current, 'fetching changes')
sha_to_run, rt = get_new_commits(current)
if sha_to_run is not None and len(sha_to_run) is 0:
print "No new SHA1, got HTTP status:", rt
print("No new SHA1, got HTTP status:", rt)
sys_exit_safe()
elif sha_to_run is None:
print "Something went badly wrong, unable to use Github compare"
print("Something went badly wrong, unable to use Github compare")
sys_exit_safe()
else:
# Ok, we do not have an existing SHA1, let us get one
print 'No pre-existing SHA1, fetching refs'
print('No pre-existing SHA1, fetching refs')
sha1, rt = get_current_sha1()
if sha1 is None:
print "No SHA1, got HTTP status:", rt
print("No SHA1, got HTTP status:", rt)
sys_exit_safe()
sha_to_run = [ sha1 ]
else:
print "Using forced SHA1 from env"
print("Using forced SHA1 from env")
print "Will execute for", sha_to_run
print("Will execute for", sha_to_run)
for sha in sha_to_run:
if not ensure_git_clone(sha):
print "Error with git repo handling."
print("Error with git repo handling.")
sys_exit_safe()
print "Ready for", sha
print("Ready for", sha)
print "Let us place ourselves into the git clone directory ..."
print("Let us place ourselves into the git clone directory ...")
root_dir = os.getcwd()
os.chdir(DEEPSPEECH_CLONE_PATH)
print "Copy previous run logs from backup"
print("Copy previous run logs from backup")
populate_previous_logs()
print "Starting GPU nvidia-smi monitoring"
print("Starting GPU nvidia-smi monitoring")
gpu_usage_csv, gpu_usage_charts = ensure_gpu_usage(root_dir)
gu = GPUUsage(csvfile=gpu_usage_csv)
gu.start()
print "Do the training for getting WER computation"
print("Do the training for getting WER computation")
exec_wer_run()
print "Producing GPU monitoring charts"
print("Producing GPU monitoring charts")
gu.join(5.0)
gu.stop()
GPUUsageChart(source=gpu_usage_csv, basename=gpu_usage_charts)
print "Backup the logs we just produced"
print("Backup the logs we just produced")
save_logs()
print "Save progress"
print("Save progress")
write_last_sha1(sha)
print "Let us place back to the previous directory %s ..." % root_dir
print("Let us place back to the previous directory %s ..." % root_dir)
os.chdir(root_dir)
print "Getting rid of git clone"
print("Getting rid of git clone")
wipe_git_clone()
release_lock()

1
util/importers/LDC97S62.py
View File

@ -1,3 +1,4 @@
from __future__ import print_function
import fnmatch
import numpy as np
import os

1
util/importers/fisher.py
View File

@ -1,3 +1,4 @@
from __future__ import print_function
import fnmatch
import os
import subprocess

1
util/importers/librivox.py
View File

@ -1,3 +1,4 @@
from __future__ import print_function
import codecs
import fnmatch
import os

11
util/shared_lib.py
View File

@ -1,3 +1,4 @@
from __future__ import print_function
from gpu import get_available_gpus
from ctypes import cdll
from sys import platform as _platform
@ -17,11 +18,11 @@ def check_cupti():
libname = get_cupti_libname()
cupti = check_so(libname)
if cupti is None:
print "INFO: No %s because no GPU, go ahead." % libname
print("INFO: No %s because no GPU, go ahead." % libname)
elif cupti is True:
print "INFO: Found %s." % libname
print("INFO: Found %s." % libname)
else:
print "WARNING: Running on GPU but no %s could be found ; will be unable to report GPU VRAM usage." % libname
print("WARNING: Running on GPU but no %s could be found ; will be unable to report GPU VRAM usage." % libname)
def check_so(soname):
"""
@ -35,10 +36,10 @@ def check_so(soname):
# Try to force load lib, this would fail if the lib is not there :)
try:
lib = cdll.LoadLibrary(soname)
print "INFO: Found so as", lib
print("INFO: Found so as", lib)
assert lib.__class__.__name__ == 'CDLL'
assert lib._name == soname
return True
except OSError as ex:
print "WARNING:", ex
print("WARNING:", ex)
return False

45
util/website.py
View File

@ -1,3 +1,4 @@
from __future__ import print_function
import os
import paramiko
import pysftp
@ -24,7 +25,7 @@ def parse_for_deps(filename):
all_resources = map(lambda x: os.stat(x), external_links + external_scripts)
except OSError as ex:
if ex.errno == 2:
print "Missing dependency", ex
print("Missing dependency", ex)
return []
raise ex
@ -33,7 +34,7 @@ def parse_for_deps(filename):
readable_resources = map(lambda x: os.open(x, os.O_RDONLY), external_links + external_scripts)
map(lambda x: os.close(x), readable_resources)
except OSError as ex:
print "Unreadable dependency", ex
print("Unreadable dependency", ex)
return []
# It should be all good.
@ -55,7 +56,7 @@ def verify_ssh_dir(auth_infos):
we want to push data is either empty or contains at least a .htaccess and
index.htm file
"""
print "Checking connection: %s@%s:%s (port %d)" % (auth_infos['ds_website_username'], auth_infos['ds_website_server_fqdn'], auth_infos['ds_website_server_root'], auth_infos['ds_website_server_port'])
print("Checking connection: %s@%s:%s (port %d)" % (auth_infos['ds_website_username'], auth_infos['ds_website_server_fqdn'], auth_infos['ds_website_server_root'], auth_infos['ds_website_server_port']))
try:
with get_ssh(auth_infos) as sftp:
@ -64,13 +65,13 @@ def verify_ssh_dir(auth_infos):
if len(files) == 0 or (('.htaccess' in files) and ('index.htm' in files)):
return True
else:
print "Invalid content for %s:" % (auth_infos['ds_website_server_root']), files
print("Invalid content for %s:" % (auth_infos['ds_website_server_root']), files)
return False
except paramiko.ssh_exception.AuthenticationException as ex:
print "Authentication error, please verify credentials"
print("Authentication error, please verify credentials")
return False
except IOError as ex:
print "Unable to read a file (private key or invalid path on server ?)", ex
print("Unable to read a file (private key or invalid path on server ?)", ex)
return False
# Should not be reached
@ -93,23 +94,23 @@ def push_files_sftp(files, auth_infos):
# Create dirs if needed, they all should depend from the root
for dir in dirs:
if not sftp.isdir(dir):
print "Creating directory", dir
print("Creating directory", dir)
sftp.makedirs(dir)
created.append(dir)
# Push all files, chdir() for each
for fil in files:
with sftp.cd(os.path.dirname(fil)):
print "Pushing", fil
print("Pushing", fil)
sftp.put(fil)
pushed.append(fil)
return pushed
except paramiko.ssh_exception.AuthenticationException as ex:
print "Authentication error, please verify credentials"
print("Authentication error, please verify credentials")
return False
except IOError as ex:
print "Unable to read a file (private key or invalid path on server ?)", ex
print("Unable to read a file (private key or invalid path on server ?)", ex)
return False
# Should not be reached
@ -145,12 +146,12 @@ def maybe_publish(file='index.htm'):
try:
ssh_auth_infos[key] = int(os.environ.get(key))
except TypeError as ex:
print "WARNING:", "Keeping default SSH port value because of:", ex
print("WARNING:", "Keeping default SSH port value because of:", ex)
missing_env = filter(lambda x: len(str(ssh_auth_infos[x])) == 0, ssh_auth_infos.keys())
if len(missing_env) > 0:
print "Not publishing, missing some required environment variables:", missing_env
print "But maybe this is what you wanted, after all ..."
print("Not publishing, missing some required environment variables:", missing_env)
print("But maybe this is what you wanted, after all ...")
return False
merge_logs("logs")
@ -158,32 +159,32 @@ def maybe_publish(file='index.htm'):
all_deps = parse_for_deps(file)
if len(all_deps) == 0:
print "Problem during deps computation, aborting"
print("Problem during deps computation, aborting")
return False
if not verify_ssh_dir(ssh_auth_infos):
print "Problem during SSH directory verification, aborting"
print("Problem during SSH directory verification, aborting")
return False
all_files = [ file ] + all_deps
uploaded = push_files_sftp(all_files, ssh_auth_infos)
if len(uploaded) == 0:
print "Unable to upload anything"
print("Unable to upload anything")
return False
elif len(uploaded) != len(all_files):
print "Partial upload has been completed:"
print "all_files=", all_files
print "uploaded=", uploaded
print("Partial upload has been completed:")
print("all_files=", all_files)
print("uploaded=", uploaded)
else:
print "Complete upload has been completed."
print("Complete upload has been completed.")
return True
# Support CLI invocation
if __name__ == "__main__":
if maybe_publish():
print "All good!"
print("All good!")
sys.exit(0)
else:
print "Error happened ..."
print("Error happened ...")
sys.exit(1)