From 781880ed93debc2dc27da2e3a5ed38e0445e7450 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Tue, 23 Mar 2021 12:26:33 -0600 Subject: [PATCH] [Script] Add tolerance options to check qor script --- openfpga_flow/scripts/check_qor.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/openfpga_flow/scripts/check_qor.py b/openfpga_flow/scripts/check_qor.py index 70efd99ac..9b55e64c1 100644 --- a/openfpga_flow/scripts/check_qor.py +++ b/openfpga_flow/scripts/check_qor.py @@ -37,6 +37,9 @@ parser.add_argument('--reference_csv_file', required=True, help='Specify the reference csv file constaining flow-run information') parser.add_argument('--metric_checklist_csv_file', required=True, help='Specify the csv file constaining metrics to be checked') +# By default, allow a 50% tolerance when checking metrics +parser.add_argument('--check_tolerance', default="0.5,1.5", + help='Specify the tolerance when checking metrics. Format ,') args = parser.parse_args() ##################################################################### @@ -77,6 +80,12 @@ ref_results = {} for row in ref_csv_content: ref_results[row[csv_name_tag]] = row; +##################################################################### +# Parse the tolerance to be applied when checking metrics +##################################################################### +lower_bound_factor = float(args.check_tolerance.split(",")[0]) +upper_bound_factor = float(args.check_tolerance.split(",")[1]) + ##################################################################### # Parse the csv file to check ##################################################################### @@ -87,9 +96,10 @@ with open(args.check_csv_file, newline='') as check_csv_file: for row in results_to_check: # Start from line 1 and check information for metric_to_check in metric_checklist: - if (ref_results[row[csv_name_tag]][metric_to_check] > row[metric_to_check]): + # Check if the metric is in a range + if (lower_bound_factor * float(ref_results[row[csv_name_tag]][metric_to_check]) > float(row[metric_to_check])) or (upper_bound_factor * float(ref_results[row[csv_name_tag]][metric_to_check]) < float(row[metric_to_check])) : # Check QoR failed, error out - logging.error("Benchmark " + str(row[csv_name_tag]) + " failed in checking '" + str(metric_to_check) +"'\n" + "Found: " + str(row[metric_to_check]) + " but expected: " + str(ref_results[row[csv_name_tag]][metric_to_check])) + logging.error("Benchmark " + str(row[csv_name_tag]) + " failed in checking '" + str(metric_to_check) +"'\n" + "Found: " + str(row[metric_to_check]) + " but expected: " + str(ref_results[row[csv_name_tag]][metric_to_check]) + " outside range [" + str(lower_bound_factor * 100) + "%, " + str(upper_bound_factor * 100) + "%]") check_error_count += 1 # Pass this metric check, increase counter checkpoint_count += 1