diff --git a/util/automation.py b/util/automation.py index 91e14544..721e70a0 100644 --- a/util/automation.py +++ b/util/automation.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +from __future__ import print_function import json import os import git @@ -65,21 +66,21 @@ class GPUUsage(Thread): def stop(self): if not self._process: - print "Trying to stop nvidia-smi but no more process, please fix." + print("Trying to stop nvidia-smi but no more process, please fix.") return - print "Ending nvidia-smi monitoring: PID", self._process.pid + print("Ending nvidia-smi monitoring: PID", self._process.pid) self._process.terminate() - print "Ended nvidia-smi monitoring ..." + print("Ended nvidia-smi monitoring ...") def run(self): - print "Starting nvidia-smi monitoring" + print("Starting nvidia-smi monitoring") # If the system has no CUDA setup, then this will fail. try: self._process = subprocess.Popen(self._cmd, stdout=subprocess.PIPE) except OSError as ex: - print "Unable to start monitoring, check your environment:", ex + print("Unable to start monitoring, check your environment:", ex) return writer = None @@ -130,7 +131,7 @@ class GPUUsageChart(): for plot in self._rows: self.produce_plot(plot) except IOError as ex: - print "Unable to read", ex + print("Unable to read", ex) def append_data(self, row): for bucket, value in row.iteritems(): @@ -147,18 +148,18 @@ class GPUUsageChart(): self._data[bucket][gpu] += [ value ] def read(self): - print "Reading data from", self._csv + print("Reading data from", self._csv) with open(self._csv, 'r') as f: for r in csv.DictReader(f): self.append_data(r) def produce_plot(self, key, with_spline=True): png = self._basename % (key, ) - print "Producing plot for", key, "as", png + print("Producing plot for", key, "as", png) fig, axis = plt.subplots() data = self._data[key] if data is None: - print "Data was empty, aborting" + print("Data was empty, aborting") return x = range(len(data[0])) @@ -339,10 +340,10 @@ def ensure_git_clone(sha): # If non-existent, create a clone if not os.path.isdir(gitdir): source = get_github_repo_url() - print "Performing a fresh clone from", source, "to", DEEPSPEECH_CLONE_PATH + print("Performing a fresh clone from", source, "to", DEEPSPEECH_CLONE_PATH) ds_repo = git.Repo.clone_from(source, DEEPSPEECH_CLONE_PATH) else: - print "Using existing clone from", DEEPSPEECH_CLONE_PATH + print("Using existing clone from", DEEPSPEECH_CLONE_PATH) ds_repo = git.Repo(gitdir) # Ensure we have a valid non bare local clone @@ -412,7 +413,7 @@ def ensure_gpu_usage(root_dir): gpu_usage_root = os.path.abspath(os.environ.get('ds_gpu_usage_root', root_dir)) gpu_usage_path = os.path.join(gpu_usage_root, get_git_desc()) - print "Will produce CSV and charts in %s" % gpu_usage_path + print("Will produce CSV and charts in %s" % gpu_usage_path) if not os.path.isdir(gpu_usage_path): os.makedirs(gpu_usage_path) @@ -451,9 +452,9 @@ def ensure_checkpoint_directory(): # it does not exists. # If it actually exists, the later os.path.isdir(checkpoint_dir) will # take care of setting the checkpoint_restore value - print "Trying with maybe_checkpoint_dir", maybe_checkpoint_dir + print("Trying with maybe_checkpoint_dir", maybe_checkpoint_dir) if not os.path.isdir(maybe_checkpoint_dir): - print "Trying to force-restore checkpoint from non-existent directory", maybe_checkpoint_dir + print("Trying to force-restore checkpoint from non-existent directory", maybe_checkpoint_dir) else: checkpoint_dir = maybe_checkpoint_dir @@ -461,10 +462,10 @@ def ensure_checkpoint_directory(): if not os.path.isdir(checkpoint_dir): os.makedirs(checkpoint_dir) else: - print "Found existing checkpoint dir, re-using it" + print("Found existing checkpoint dir, re-using it") checkpoint_restore = True - print "Ensured checkpoint infos: ", checkpoint_dir, checkpoint_restore + print("Ensured checkpoint infos: ", checkpoint_dir, checkpoint_restore) return checkpoint_dir, checkpoint_restore @@ -533,64 +534,64 @@ def exec_main(): if len(sha_to_run) == 0: current = get_last_sha1() if len(current) == 40: - print 'Existing SHA1:', current, 'fetching changes' + print('Existing SHA1:', current, 'fetching changes') sha_to_run, rt = get_new_commits(current) if sha_to_run is not None and len(sha_to_run) is 0: - print "No new SHA1, got HTTP status:", rt + print("No new SHA1, got HTTP status:", rt) sys_exit_safe() elif sha_to_run is None: - print "Something went badly wrong, unable to use Github compare" + print("Something went badly wrong, unable to use Github compare") sys_exit_safe() else: # Ok, we do not have an existing SHA1, let us get one - print 'No pre-existing SHA1, fetching refs' + print('No pre-existing SHA1, fetching refs') sha1, rt = get_current_sha1() if sha1 is None: - print "No SHA1, got HTTP status:", rt + print("No SHA1, got HTTP status:", rt) sys_exit_safe() sha_to_run = [ sha1 ] else: - print "Using forced SHA1 from env" + print("Using forced SHA1 from env") - print "Will execute for", sha_to_run + print("Will execute for", sha_to_run) for sha in sha_to_run: if not ensure_git_clone(sha): - print "Error with git repo handling." + print("Error with git repo handling.") sys_exit_safe() - print "Ready for", sha + print("Ready for", sha) - print "Let us place ourselves into the git clone directory ..." + print("Let us place ourselves into the git clone directory ...") root_dir = os.getcwd() os.chdir(DEEPSPEECH_CLONE_PATH) - print "Copy previous run logs from backup" + print("Copy previous run logs from backup") populate_previous_logs() - print "Starting GPU nvidia-smi monitoring" + print("Starting GPU nvidia-smi monitoring") gpu_usage_csv, gpu_usage_charts = ensure_gpu_usage(root_dir) gu = GPUUsage(csvfile=gpu_usage_csv) gu.start() - print "Do the training for getting WER computation" + print("Do the training for getting WER computation") exec_wer_run() - print "Producing GPU monitoring charts" + print("Producing GPU monitoring charts") gu.join(5.0) gu.stop() GPUUsageChart(source=gpu_usage_csv, basename=gpu_usage_charts) - print "Backup the logs we just produced" + print("Backup the logs we just produced") save_logs() - print "Save progress" + print("Save progress") write_last_sha1(sha) - print "Let us place back to the previous directory %s ..." % root_dir + print("Let us place back to the previous directory %s ..." % root_dir) os.chdir(root_dir) - print "Getting rid of git clone" + print("Getting rid of git clone") wipe_git_clone() release_lock() diff --git a/util/importers/LDC97S62.py b/util/importers/LDC97S62.py index 8a605e77..f516e9ca 100644 --- a/util/importers/LDC97S62.py +++ b/util/importers/LDC97S62.py @@ -1,3 +1,4 @@ +from __future__ import print_function import fnmatch import numpy as np import os diff --git a/util/importers/fisher.py b/util/importers/fisher.py index 76f7f4e4..60164fcc 100644 --- a/util/importers/fisher.py +++ b/util/importers/fisher.py @@ -1,3 +1,4 @@ +from __future__ import print_function import fnmatch import os import subprocess diff --git a/util/importers/librivox.py b/util/importers/librivox.py index 899c3e81..1edc3a17 100644 --- a/util/importers/librivox.py +++ b/util/importers/librivox.py @@ -1,3 +1,4 @@ +from __future__ import print_function import codecs import fnmatch import os diff --git a/util/shared_lib.py b/util/shared_lib.py index ab32ff26..c2bb104b 100644 --- a/util/shared_lib.py +++ b/util/shared_lib.py @@ -1,3 +1,4 @@ +from __future__ import print_function from gpu import get_available_gpus from ctypes import cdll from sys import platform as _platform @@ -17,11 +18,11 @@ def check_cupti(): libname = get_cupti_libname() cupti = check_so(libname) if cupti is None: - print "INFO: No %s because no GPU, go ahead." % libname + print("INFO: No %s because no GPU, go ahead." % libname) elif cupti is True: - print "INFO: Found %s." % libname + print("INFO: Found %s." % libname) else: - print "WARNING: Running on GPU but no %s could be found ; will be unable to report GPU VRAM usage." % libname + print("WARNING: Running on GPU but no %s could be found ; will be unable to report GPU VRAM usage." % libname) def check_so(soname): """ @@ -35,10 +36,10 @@ def check_so(soname): # Try to force load lib, this would fail if the lib is not there :) try: lib = cdll.LoadLibrary(soname) - print "INFO: Found so as", lib + print("INFO: Found so as", lib) assert lib.__class__.__name__ == 'CDLL' assert lib._name == soname return True except OSError as ex: - print "WARNING:", ex + print("WARNING:", ex) return False diff --git a/util/website.py b/util/website.py index 1b14010d..0435ff79 100644 --- a/util/website.py +++ b/util/website.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import paramiko import pysftp @@ -24,7 +25,7 @@ def parse_for_deps(filename): all_resources = map(lambda x: os.stat(x), external_links + external_scripts) except OSError as ex: if ex.errno == 2: - print "Missing dependency", ex + print("Missing dependency", ex) return [] raise ex @@ -33,7 +34,7 @@ def parse_for_deps(filename): readable_resources = map(lambda x: os.open(x, os.O_RDONLY), external_links + external_scripts) map(lambda x: os.close(x), readable_resources) except OSError as ex: - print "Unreadable dependency", ex + print("Unreadable dependency", ex) return [] # It should be all good. @@ -55,7 +56,7 @@ def verify_ssh_dir(auth_infos): we want to push data is either empty or contains at least a .htaccess and index.htm file """ - print "Checking connection: %s@%s:%s (port %d)" % (auth_infos['ds_website_username'], auth_infos['ds_website_server_fqdn'], auth_infos['ds_website_server_root'], auth_infos['ds_website_server_port']) + print("Checking connection: %s@%s:%s (port %d)" % (auth_infos['ds_website_username'], auth_infos['ds_website_server_fqdn'], auth_infos['ds_website_server_root'], auth_infos['ds_website_server_port'])) try: with get_ssh(auth_infos) as sftp: @@ -64,13 +65,13 @@ def verify_ssh_dir(auth_infos): if len(files) == 0 or (('.htaccess' in files) and ('index.htm' in files)): return True else: - print "Invalid content for %s:" % (auth_infos['ds_website_server_root']), files + print("Invalid content for %s:" % (auth_infos['ds_website_server_root']), files) return False except paramiko.ssh_exception.AuthenticationException as ex: - print "Authentication error, please verify credentials" + print("Authentication error, please verify credentials") return False except IOError as ex: - print "Unable to read a file (private key or invalid path on server ?)", ex + print("Unable to read a file (private key or invalid path on server ?)", ex) return False # Should not be reached @@ -93,23 +94,23 @@ def push_files_sftp(files, auth_infos): # Create dirs if needed, they all should depend from the root for dir in dirs: if not sftp.isdir(dir): - print "Creating directory", dir + print("Creating directory", dir) sftp.makedirs(dir) created.append(dir) # Push all files, chdir() for each for fil in files: with sftp.cd(os.path.dirname(fil)): - print "Pushing", fil + print("Pushing", fil) sftp.put(fil) pushed.append(fil) return pushed except paramiko.ssh_exception.AuthenticationException as ex: - print "Authentication error, please verify credentials" + print("Authentication error, please verify credentials") return False except IOError as ex: - print "Unable to read a file (private key or invalid path on server ?)", ex + print("Unable to read a file (private key or invalid path on server ?)", ex) return False # Should not be reached @@ -145,12 +146,12 @@ def maybe_publish(file='index.htm'): try: ssh_auth_infos[key] = int(os.environ.get(key)) except TypeError as ex: - print "WARNING:", "Keeping default SSH port value because of:", ex + print("WARNING:", "Keeping default SSH port value because of:", ex) missing_env = filter(lambda x: len(str(ssh_auth_infos[x])) == 0, ssh_auth_infos.keys()) if len(missing_env) > 0: - print "Not publishing, missing some required environment variables:", missing_env - print "But maybe this is what you wanted, after all ..." + print("Not publishing, missing some required environment variables:", missing_env) + print("But maybe this is what you wanted, after all ...") return False merge_logs("logs") @@ -158,32 +159,32 @@ def maybe_publish(file='index.htm'): all_deps = parse_for_deps(file) if len(all_deps) == 0: - print "Problem during deps computation, aborting" + print("Problem during deps computation, aborting") return False if not verify_ssh_dir(ssh_auth_infos): - print "Problem during SSH directory verification, aborting" + print("Problem during SSH directory verification, aborting") return False all_files = [ file ] + all_deps uploaded = push_files_sftp(all_files, ssh_auth_infos) if len(uploaded) == 0: - print "Unable to upload anything" + print("Unable to upload anything") return False elif len(uploaded) != len(all_files): - print "Partial upload has been completed:" - print "all_files=", all_files - print "uploaded=", uploaded + print("Partial upload has been completed:") + print("all_files=", all_files) + print("uploaded=", uploaded) else: - print "Complete upload has been completed." + print("Complete upload has been completed.") return True # Support CLI invocation if __name__ == "__main__": if maybe_publish(): - print "All good!" + print("All good!") sys.exit(0) else: - print "Error happened ..." + print("Error happened ...") sys.exit(1)