From 47020670b64b4cc474799a5f7d8c5f72a28020a5 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 22 Jun 2016 17:11:01 +0200 Subject: [PATCH 1/3] test-zstd-speed.py: added --message in start and finish e-mail - support for ~ in filenames - result files ending with .txt --- tests/test-zstd-speed.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 2382f231e..5444c1da2 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -168,7 +168,7 @@ def check_commit(branch, commit, args, testFilePaths, have_mutt, have_mail): results_files = "" for filePath in testFilePaths: fileName = filePath.rpartition('/')[2] - resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName + resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" last_commit, cspeed, dspeed = get_last_results(resultsFileName) if not args.dry_run: @@ -205,6 +205,7 @@ if __name__ == '__main__': testFileNames = args.testFileNames.split() testFilePaths = [] for fileName in testFileNames: + fileName = os.path.expanduser(fileName) if os.path.isfile(fileName): testFilePaths.append(os.path.abspath(fileName)) else: @@ -248,7 +249,7 @@ if __name__ == '__main__': log("ERROR: %s already exists, exiting" % pidfile) exit(1) - send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, '', have_mutt, have_mail) + send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail) file(pidfile, 'w').write(pid) while True: @@ -266,5 +267,5 @@ if __name__ == '__main__': time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) - send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, '', have_mutt, have_mail) + send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail) exit(0) From 116128ce4134f387efad372d295630348f928729 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 22 Jun 2016 18:12:57 +0200 Subject: [PATCH 2/3] test-zstd-speed.py: improved exception handling --- tests/test-zstd-speed.py | 74 ++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 5444c1da2..2ef4d6ba5 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -123,7 +123,7 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime)) time.sleep(sleepTime) start_load = str(os.getloadavg()) - result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath, print_output=True) + result = execute('programs/zstd -qi5b1e%s %s' % (lastCLevel, testFilePath), print_output=True) end_load = str(os.getloadavg()) linesExpected = lastCLevel + 2; if len(result) != linesExpected: @@ -148,44 +148,35 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP return text -def check_commit(branch, commit, args, testFilePaths, have_mutt, have_mail): - try: - last_commit = None - commitFileName = working_path + "/commit_" + branch.replace("/", "_") - if os.path.isfile(commitFileName): - last_commit = file(commitFileName, 'r').read() - file(commitFileName, 'w').write(commit) +def update_config_file(branch, commit): + last_commit = None + commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt" + if os.path.isfile(commitFileName): + last_commit = file(commitFileName, 'r').read() + file(commitFileName, 'w').write(commit) + return last_commit - if commit == last_commit: - log("skipping branch %s: head %s already processed" % (branch, commit)) - return - log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) - compile(branch, commit, last_commit, args.dry_run) +def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): + compile(branch, commit, last_commit, args.dry_run) + logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" + text_to_send = [] + results_files = "" + for filePath in testFilePaths: + fileName = filePath.rpartition('/')[2] + resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + last_commit, cspeed, dspeed = get_last_results(resultsFileName) - logFileName = working_path + "/log_" + branch.replace("/", "_") - text_to_send = [] - results_files = "" - for filePath in testFilePaths: - fileName = filePath.rpartition('/')[2] - resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - last_commit, cspeed, dspeed = get_last_results(resultsFileName) - - if not args.dry_run: + if not args.dry_run: + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) + if text: + log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) if text: - log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) - text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - if text_to_send: - send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail) - except Exception as e: - stack = traceback.format_exc() - email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit) - send_email(args.emails, email_topic, stack, have_mutt, have_mail) - print(stack) + text_to_send.append(text) + results_files += resultsFileName + " " + if text_to_send: + send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail) if __name__ == '__main__': @@ -258,13 +249,22 @@ if __name__ == '__main__': if (loadavg <= args.maxLoadAvg): branches = git_get_branches() for branch in branches: - commits = execute('git show -s --format=%h ' + branch)[0] - for commit in [commits]: - check_commit(branch, commit, args, testFilePaths, have_mutt, have_mail) + commit = execute('git show -s --format=%h ' + branch)[0] + last_commit = update_config_file(branch, commit) + if commit == last_commit: + log("skipping branch %s: head %s already processed" % (branch, commit)) + else: + log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) + test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail) else: log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) log("sleep for %s seconds" % args.sleepTime) time.sleep(args.sleepTime) + except Exception as e: + stack = traceback.format_exc() + email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit) + send_email(args.emails, email_topic, stack, have_mutt, have_mail) + print(stack) except KeyboardInterrupt: os.unlink(pidfile) send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail) From 82babfc0c6aa128f175c3a0c7d6e839dbcafbdd8 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 22 Jun 2016 20:06:42 +0200 Subject: [PATCH 3/3] test-zstd-speed.py: reordering of functions --- tests/test-zstd-speed.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 2ef4d6ba5..522227a4e 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -88,16 +88,6 @@ def git_get_changes(branch, commit, last_commit): return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits) -def compile(branch, commit, last_commit, dry_run): - local_branch = string.split(branch, '/')[1] - version = local_branch.rpartition('-')[2] - version = version + '_' + commit - execute('git checkout -- . && git checkout ' + branch) - print(git_get_changes(branch, commit, last_commit)) - if not dry_run: - execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) - - def get_last_results(resultsFileName): if not os.path.isfile(resultsFileName): return None, None, None @@ -158,7 +148,10 @@ def update_config_file(branch, commit): def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): - compile(branch, commit, last_commit, args.dry_run) + local_branch = string.split(branch, '/')[1] + version = local_branch.rpartition('-')[2] + '_' + commit + if not args.dry_run: + execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" @@ -166,7 +159,6 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav fileName = filePath.rpartition('/')[2] resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" last_commit, cspeed, dspeed = get_last_results(resultsFileName) - if not args.dry_run: text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) if text: @@ -255,6 +247,8 @@ if __name__ == '__main__': log("skipping branch %s: head %s already processed" % (branch, commit)) else: log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) + execute('git checkout -- . && git checkout ' + branch) + print(git_get_changes(branch, commit, last_commit)) test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail) else: log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))