From 6719a9aa26b310b5283e7467e398d9446278acf8 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:03:01 +0800 Subject: [PATCH 01/13] [test] update golden netlists/testbenches etc. --- .../and2_formal_random_top_tb.v | 33 +++--- .../and2_top_formal_verification.v | 112 +++++++++--------- .../bitstream_distribution.xml | 66 ++++++----- 3 files changed, 111 insertions(+), 100 deletions(-) diff --git a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_formal_random_top_tb.v b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_formal_random_top_tb.v index 9f3873a0c..cc793331a 100644 --- a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_formal_random_top_tb.v +++ b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_formal_random_top_tb.v @@ -19,29 +19,31 @@ module and2_top_formal_verification_random_tb; reg [0:0] b; // ----- FPGA fabric outputs ------- - wire [0:0] out_c_gfpga; + wire [0:0] c_gfpga; // ----- Benchmark outputs ------- - wire [0:0] out_c_bench; + wire [0:0] c_bench; // ----- Output vectors checking flags ------- - reg [0:0] out_c_flag; + reg [0:0] c_flag; // ----- Error counter ------- integer nb_error= 0; // ----- FPGA fabric instanciation ------- and2_top_formal_verification FPGA_DUT( - .a_fm(a), - .b_fm(b), - .out_c_fm(out_c_gfpga) ); + .a(a), + .b(b), + .c(c_gfpga) + ); // ----- End FPGA Fabric Instanication ------- // ----- Reference Benchmark Instanication ------- and2 REF_DUT( .a(a), .b(b), - .c(out_c_bench) ); + .c(c_bench) + ); // ----- End reference Benchmark Instanication ------- // ----- Clock 'clk' Initialization ------- @@ -61,7 +63,7 @@ module and2_top_formal_verification_random_tb; a <= 1'b0; b <= 1'b0; - out_c_flag[0] <= 1'b0; + c_flag[0] <= 1'b0; end // ----- Input Stimulus ------- @@ -77,19 +79,20 @@ module and2_top_formal_verification_random_tb; always@(negedge clk[0]) begin if (1'b1 == sim_start[0]) begin sim_start[0] <= ~sim_start[0]; - end else begin - if(!(out_c_gfpga === out_c_bench) && !(out_c_bench === 1'bx)) begin - out_c_flag <= 1'b1; + end else +begin + if(!(c_gfpga === c_bench) && !(c_bench === 1'bx)) begin + c_flag <= 1'b1; end else begin - out_c_flag<= 1'b0; + c_flag<= 1'b0; end end end - always@(posedge out_c_flag) begin - if(out_c_flag) begin + always@(posedge c_flag) begin + if(c_flag) begin nb_error = nb_error + 1; - $display("Mismatch on out_c_gfpga at time = %t", $realtime); + $display("Mismatch on c_gfpga at time = %t", $realtime); end end diff --git a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_top_formal_verification.v b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_top_formal_verification.v index 3d675c1c4..1334fe5d8 100644 --- a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_top_formal_verification.v +++ b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/and2_top_formal_verification.v @@ -11,76 +11,76 @@ `default_nettype none module and2_top_formal_verification ( -input [0:0] a_fm, -input [0:0] b_fm, -output [0:0] out_c_fm); +input [0:0] a, +input [0:0] b, +output [0:0] c); // ----- Local wires for FPGA fabric ----- -wire [0:31] gfpga_pad_GPIO_PAD; -wire [0:0] ccff_head; -wire [0:0] ccff_tail; -wire [0:0] prog_clk; -wire [0:0] set; -wire [0:0] reset; -wire [0:0] clk; +wire [0:31] gfpga_pad_GPIO_PAD_fm; +wire [0:0] ccff_head_fm; +wire [0:0] ccff_tail_fm; +wire [0:0] prog_clk_fm; +wire [0:0] set_fm; +wire [0:0] reset_fm; +wire [0:0] clk_fm; // ----- FPGA top-level module to be capsulated ----- fpga_top U0_formal_verification ( - .prog_clk(prog_clk[0]), - .set(set[0]), - .reset(reset[0]), - .clk(clk[0]), - .gfpga_pad_GPIO_PAD(gfpga_pad_GPIO_PAD[0:31]), - .ccff_head(ccff_head[0]), - .ccff_tail(ccff_tail[0])); + .prog_clk(prog_clk_fm[0]), + .set(set_fm[0]), + .reset(reset_fm[0]), + .clk(clk_fm[0]), + .gfpga_pad_GPIO_PAD(gfpga_pad_GPIO_PAD_fm[0:31]), + .ccff_head(ccff_head_fm[0]), + .ccff_tail(ccff_tail_fm[0])); // ----- Begin Connect Global ports of FPGA top module ----- - assign set[0] = 1'b0; - assign reset[0] = 1'b0; - assign clk[0] = 1'b0; - assign prog_clk[0] = 1'b0; + assign set_fm[0] = 1'b0; + assign reset_fm[0] = 1'b0; + assign clk_fm[0] = 1'b0; + assign prog_clk_fm[0] = 1'b0; // ----- End Connect Global ports of FPGA top module ----- // ----- Link BLIF Benchmark I/Os to FPGA I/Os ----- -// ----- Blif Benchmark input a is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD[6] ----- - assign gfpga_pad_GPIO_PAD[6] = a_fm[0]; +// ----- Blif Benchmark input a is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD_fm[6] ----- + assign gfpga_pad_GPIO_PAD_fm[6] = a[0]; -// ----- Blif Benchmark input b is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD[1] ----- - assign gfpga_pad_GPIO_PAD[1] = b_fm[0]; +// ----- Blif Benchmark input b is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD_fm[1] ----- + assign gfpga_pad_GPIO_PAD_fm[1] = b[0]; -// ----- Blif Benchmark output out_c is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD[9] ----- - assign out_c_fm[0] = gfpga_pad_GPIO_PAD[9]; +// ----- Blif Benchmark output c is mapped to FPGA IOPAD gfpga_pad_GPIO_PAD_fm[9] ----- + assign c[0] = gfpga_pad_GPIO_PAD_fm[9]; // ----- Wire unused FPGA I/Os to constants ----- - assign gfpga_pad_GPIO_PAD[0] = 1'b0; - assign gfpga_pad_GPIO_PAD[2] = 1'b0; - assign gfpga_pad_GPIO_PAD[3] = 1'b0; - assign gfpga_pad_GPIO_PAD[4] = 1'b0; - assign gfpga_pad_GPIO_PAD[5] = 1'b0; - assign gfpga_pad_GPIO_PAD[7] = 1'b0; - assign gfpga_pad_GPIO_PAD[8] = 1'b0; - assign gfpga_pad_GPIO_PAD[10] = 1'b0; - assign gfpga_pad_GPIO_PAD[11] = 1'b0; - assign gfpga_pad_GPIO_PAD[12] = 1'b0; - assign gfpga_pad_GPIO_PAD[13] = 1'b0; - assign gfpga_pad_GPIO_PAD[14] = 1'b0; - assign gfpga_pad_GPIO_PAD[15] = 1'b0; - assign gfpga_pad_GPIO_PAD[16] = 1'b0; - assign gfpga_pad_GPIO_PAD[17] = 1'b0; - assign gfpga_pad_GPIO_PAD[18] = 1'b0; - assign gfpga_pad_GPIO_PAD[19] = 1'b0; - assign gfpga_pad_GPIO_PAD[20] = 1'b0; - assign gfpga_pad_GPIO_PAD[21] = 1'b0; - assign gfpga_pad_GPIO_PAD[22] = 1'b0; - assign gfpga_pad_GPIO_PAD[23] = 1'b0; - assign gfpga_pad_GPIO_PAD[24] = 1'b0; - assign gfpga_pad_GPIO_PAD[25] = 1'b0; - assign gfpga_pad_GPIO_PAD[26] = 1'b0; - assign gfpga_pad_GPIO_PAD[27] = 1'b0; - assign gfpga_pad_GPIO_PAD[28] = 1'b0; - assign gfpga_pad_GPIO_PAD[29] = 1'b0; - assign gfpga_pad_GPIO_PAD[30] = 1'b0; - assign gfpga_pad_GPIO_PAD[31] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[0] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[2] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[3] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[4] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[5] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[7] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[8] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[10] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[11] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[12] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[13] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[14] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[15] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[16] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[17] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[18] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[19] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[20] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[21] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[22] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[23] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[24] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[25] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[26] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[27] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[28] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[29] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[30] = 1'b0; + assign gfpga_pad_GPIO_PAD_fm[31] = 1'b0; // ----- Begin load bitstream to configuration memories ----- // ----- Begin assign bitstream to configuration memories ----- diff --git a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/bitstream_distribution.xml b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/bitstream_distribution.xml index b734007e7..34701b8fb 100644 --- a/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/bitstream_distribution.xml +++ b/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/bitstream_distribution.xml @@ -1,32 +1,40 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 751d87b8e3b9d038d622e60c53f581b0cd791f5e Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:06:47 +0800 Subject: [PATCH 02/13] [test] fix a bug in detect changes in golden netlists --- openfpga_flow/regression_test_scripts/basic_reg_test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openfpga_flow/regression_test_scripts/basic_reg_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_test.sh index 66dd32d21..fde557321 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_test.sh @@ -152,10 +152,11 @@ run-task basic_tests/explicit_multi_verilog_files --debug --show_thread_logs echo -e "Testing output files without time stamp"; run-task basic_tests/no_time_stamp --debug --show_thread_logs # Run git-diff to ensure no changes on the golden netlists -if git diff --name-status -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then +git diff --name-status -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' +if git diff --name-status --exit-code -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then echo -e "Golden netlist remain unchanged" else - echo -e "Detect changes in golden scripts"; exit 1; + echo -e "Detect changes in golden netlists"; exit 1; fi echo -e "Test the remove of runtime directories" From 0dc7caf3b7e0163d873618e85856303c62de169c Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:15:39 +0800 Subject: [PATCH 03/13] [test] now regression test script supports remove all run dir through command-line options --- openfpga.sh | 4 + .../regression_test_scripts/basic_reg_test.sh | 242 +++++++++--------- .../basic_reg_yosys_only_test.sh | 5 +- .../fpga_bitstream_reg_test.sh | 22 +- .../fpga_sdc_reg_test.sh | 18 +- .../fpga_spice_reg_test.sh | 2 +- .../fpga_verilog_reg_test.sh | 98 +++---- .../iwls_benchmark_reg_test.sh | 2 +- .../micro_benchmark_reg_test.sh | 12 +- .../quicklogic_reg_test.sh | 8 +- .../vtr_benchmark_reg_test.sh | 2 +- 11 files changed, 211 insertions(+), 204 deletions(-) diff --git a/openfpga.sh b/openfpga.sh index e4f08cd41..e31171127 100755 --- a/openfpga.sh +++ b/openfpga.sh @@ -65,6 +65,10 @@ clean-run () { rm -rf ./openfpga_flow/**/run??? } +run-task-run () { + $PYTHON_EXEC $OPENFPGA_SCRIPT_PATH/run_fpga_task.py --remove_run_dir all +} + run-modelsim () { $PYTHON_EXEC $OPENFPGA_SCRIPT_PATH/run_modelsim.py "$@" } diff --git a/openfpga_flow/regression_test_scripts/basic_reg_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_test.sh index fde557321..0134b21e2 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_test.sh @@ -9,148 +9,175 @@ PYTHON_EXEC=python3.8 echo -e "Basic regression tests"; echo -e "Testing configuration chain of a K4N4 FPGA"; -run-task basic_tests/full_testbench/configuration_chain --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_no_time_stamp --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_use_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_use_resetb --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_use_setb --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_use_set_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_chain_config_enable_scff --debug --show_thread_logs -run-task basic_tests/full_testbench/multi_region_configuration_chain --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_configuration_chain --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_configuration_chain_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_configuration_chain --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_multi_region_configuration_chain --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/configuration_chain --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/configuration_chain_config_done_io --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/configuration_chain_no_time_stamp --debug --show_thread_logs +run-task basic_tests/full_testbench/configuration_chain $@ +run-task basic_tests/full_testbench/configuration_chain_no_time_stamp $@ +run-task basic_tests/full_testbench/configuration_chain_use_reset $@ +run-task basic_tests/full_testbench/configuration_chain_use_resetb $@ +run-task basic_tests/full_testbench/configuration_chain_use_set $@ +run-task basic_tests/full_testbench/configuration_chain_use_setb $@ +run-task basic_tests/full_testbench/configuration_chain_use_set_reset $@ +run-task basic_tests/full_testbench/configuration_chain_config_enable_scff $@ +run-task basic_tests/full_testbench/multi_region_configuration_chain $@ +run-task basic_tests/full_testbench/fast_configuration_chain $@ +run-task basic_tests/full_testbench/fast_configuration_chain_use_set $@ +run-task basic_tests/full_testbench/smart_fast_configuration_chain $@ +run-task basic_tests/full_testbench/smart_fast_multi_region_configuration_chain $@ +run-task basic_tests/preconfig_testbench/configuration_chain $@ +run-task basic_tests/preconfig_testbench/configuration_chain_config_done_io $@ +run-task basic_tests/preconfig_testbench/configuration_chain_no_time_stamp $@ echo -e "Testing fram-based configuration protocol of a K4N4 FPGA"; -run-task basic_tests/full_testbench/configuration_frame --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_configuration_frame --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_configuration_frame --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_configuration_frame_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_ccff --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_scff --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_use_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_use_resetb --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_use_setb --debug --show_thread_logs -run-task basic_tests/full_testbench/configuration_frame_use_set_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/multi_region_configuration_frame --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_multi_region_configuration_frame --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/configuration_frame --debug --show_thread_logs +run-task basic_tests/full_testbench/configuration_frame $@ +run-task basic_tests/full_testbench/smart_fast_configuration_frame $@ +run-task basic_tests/full_testbench/fast_configuration_frame $@ +run-task basic_tests/full_testbench/fast_configuration_frame_use_set $@ +run-task basic_tests/full_testbench/configuration_frame_ccff $@ +run-task basic_tests/full_testbench/configuration_frame_scff $@ +run-task basic_tests/full_testbench/configuration_frame_use_reset $@ +run-task basic_tests/full_testbench/configuration_frame_use_resetb $@ +run-task basic_tests/full_testbench/configuration_frame_use_set $@ +run-task basic_tests/full_testbench/configuration_frame_use_setb $@ +run-task basic_tests/full_testbench/configuration_frame_use_set_reset $@ +run-task basic_tests/full_testbench/multi_region_configuration_frame $@ +run-task basic_tests/full_testbench/smart_fast_multi_region_configuration_frame $@ +run-task basic_tests/preconfig_testbench/configuration_frame $@ echo -e "Testing memory bank configuration protocol of a K4N4 FPGA"; -run-task basic_tests/full_testbench/memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/memory_bank_use_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/memory_bank_use_resetb --debug --show_thread_logs -run-task basic_tests/full_testbench/memory_bank_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/memory_bank_use_setb --debug --show_thread_logs -run-task basic_tests/full_testbench/memory_bank_use_set_reset --debug --show_thread_logs -run-task basic_tests/full_testbench/multi_region_memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/fast_memory_bank_use_set --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/smart_fast_multi_region_memory_bank --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/memory_bank --debug --show_thread_logs +run-task basic_tests/full_testbench/memory_bank $@ +run-task basic_tests/full_testbench/memory_bank_use_reset $@ +run-task basic_tests/full_testbench/memory_bank_use_resetb $@ +run-task basic_tests/full_testbench/memory_bank_use_set $@ +run-task basic_tests/full_testbench/memory_bank_use_setb $@ +run-task basic_tests/full_testbench/memory_bank_use_set_reset $@ +run-task basic_tests/full_testbench/multi_region_memory_bank $@ +run-task basic_tests/full_testbench/fast_memory_bank $@ +run-task basic_tests/full_testbench/fast_memory_bank_use_set $@ +run-task basic_tests/full_testbench/smart_fast_memory_bank $@ +run-task basic_tests/full_testbench/smart_fast_multi_region_memory_bank $@ +run-task basic_tests/preconfig_testbench/memory_bank $@ echo -e "Testing physical design friendly memory bank configuration protocol of a K4N4 FPGA"; -run-task basic_tests/full_testbench/ql_memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_use_wlr --debug --show_thread_logs -run-task basic_tests/full_testbench/multi_region_ql_memory_bank --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_flatten --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_flatten_use_wlr --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_shift_register --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_shift_register_use_wlr --debug --show_thread_logs -run-task basic_tests/full_testbench/ql_memory_bank_shift_register_multi_chain --debug --show_thread_logs +run-task basic_tests/full_testbench/ql_memory_bank $@ +run-task basic_tests/full_testbench/ql_memory_bank_use_wlr $@ +run-task basic_tests/full_testbench/multi_region_ql_memory_bank $@ +run-task basic_tests/full_testbench/ql_memory_bank_flatten $@ +run-task basic_tests/full_testbench/ql_memory_bank_flatten_use_wlr $@ +run-task basic_tests/full_testbench/ql_memory_bank_shift_register $@ +run-task basic_tests/full_testbench/ql_memory_bank_shift_register_use_wlr $@ +run-task basic_tests/full_testbench/ql_memory_bank_shift_register_multi_chain $@ echo -e "Testing testbenches without self checking features"; -run-task basic_tests/full_testbench/full_testbench_without_self_checking --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/preconfigured_testbench_without_self_checking --debug --show_thread_logs +run-task basic_tests/full_testbench/full_testbench_without_self_checking $@ +run-task basic_tests/preconfig_testbench/preconfigured_testbench_without_self_checking $@ echo -e "Testing standalone (flatten memory) configuration protocol of a K4N4 FPGA"; -run-task basic_tests/full_testbench/flatten_memory --debug --show_thread_logs -run-task basic_tests/preconfig_testbench/flatten_memory --debug --show_thread_logs +run-task basic_tests/full_testbench/flatten_memory $@ +run-task basic_tests/preconfig_testbench/flatten_memory $@ echo -e "Testing fixed device layout and routing channel width"; -run-task basic_tests/fixed_device_support --debug --show_thread_logs +run-task basic_tests/fixed_device_support $@ echo -e "Testing fabric Verilog generation only"; -run-task basic_tests/generate_fabric --debug --show_thread_logs +run-task basic_tests/generate_fabric $@ echo -e "Testing Verilog testbench generation only"; -run-task basic_tests/generate_testbench --debug --show_thread_logs +run-task basic_tests/generate_testbench $@ echo -e "Testing separated Verilog fabric netlists and testbench locations"; -run-task basic_tests/custom_fabric_netlist_location --debug --show_thread_logs +run-task basic_tests/custom_fabric_netlist_location $@ echo -e "Testing user-defined simulation settings: clock frequency and number of cycles"; -run-task basic_tests/fixed_simulation_settings/fixed_operating_clock_freq --debug --show_thread_logs +run-task basic_tests/fixed_simulation_settings/fixed_operating_clock_freq $@ # TODO: This feature is temporarily out of test due to the emergency in delivering netlists for multi-chain shift-register memory bank -#run-task basic_tests/fixed_simulation_settings/fixed_shift_register_clock_freq --debug --show_thread_logs +#run-task basic_tests/fixed_simulation_settings/fixed_shift_register_clock_freq $@ echo -e "Testing Secured FPGA fabrics"; -run-task basic_tests/fabric_key/generate_vanilla_key --debug --show_thread_logs -run-task basic_tests/fabric_key/generate_multi_region_vanilla_key --debug --show_thread_logs -run-task basic_tests/fabric_key/generate_random_key --debug --show_thread_logs -run-task basic_tests/fabric_key/generate_random_key_ql_memory_bank --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key_cc_fpga --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key_multi_region_cc_fpga --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key_qlbank_fpga --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key_multi_region_qlbank_fpga --debug --show_thread_logs -run-task basic_tests/fabric_key/load_external_key_qlbanksr_multi_chain_fpga --debug --show_thread_logs +run-task basic_tests/fabric_key/generate_vanilla_key $@ +run-task basic_tests/fabric_key/generate_multi_region_vanilla_key $@ +run-task basic_tests/fabric_key/generate_random_key $@ +run-task basic_tests/fabric_key/generate_random_key_ql_memory_bank $@ +run-task basic_tests/fabric_key/load_external_key $@ +run-task basic_tests/fabric_key/load_external_key_cc_fpga $@ +run-task basic_tests/fabric_key/load_external_key_multi_region_cc_fpga $@ +run-task basic_tests/fabric_key/load_external_key_qlbank_fpga $@ +run-task basic_tests/fabric_key/load_external_key_multi_region_qlbank_fpga $@ +run-task basic_tests/fabric_key/load_external_key_qlbanksr_multi_chain_fpga $@ # TODO: This feature is temporarily out of test due to the emergency in delivering netlists for multi-chain shift-register memory bank -#run-task basic_tests/fabric_key/load_external_key_multi_region_qlbanksr_fpga --debug --show_thread_logs +#run-task basic_tests/fabric_key/load_external_key_multi_region_qlbanksr_fpga $@ echo -e "Testing K4 series FPGA"; echo -e "Testing K4N4 with facturable LUTs"; -run-task basic_tests/k4_series/k4n4_frac_lut --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_frac_lut $@ echo -e "Testing K4N4 with asynchronous reset"; -run-task basic_tests/k4_series/k4n4_fracff --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_fracff $@ echo -e "Testing K4N4 with negative edge clocks"; -run-task basic_tests/k4_series/k4n4_fracff2edge --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_fracff2edge $@ echo -e "Testing K4N4 with hard adders"; -run-task basic_tests/k4_series/k4n4_adder --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_adder $@ echo -e "Testing K4N4 without local routing architecture"; -run-task basic_tests/k4_series/k4n4_no_local_routing --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_no_local_routing $@ echo -e "Testing K4N4 with block RAM"; -run-task basic_tests/k4_series/k4n4_bram --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_bram $@ echo -e "Testing K4N4 with LUTRAM"; -run-task basic_tests/k4_series/k4n4_lutram --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_lutram $@ echo -e "Testing K4N4 with multiple lengths of routing segments"; -run-task basic_tests/k4_series/k4n4_L124 --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_L124 $@ echo -e "Testing K4N4 with 32-bit fracturable multiplier"; -run-task basic_tests/k4_series/k4n4_frac_mult --debug --show_thread_logs +run-task basic_tests/k4_series/k4n4_frac_mult $@ echo -e "Testing K4N5 with pattern based local routing"; -run-task basic_tests/k4_series/k4n5_pattern_local_routing --debug --show_thread_logs +run-task basic_tests/k4_series/k4n5_pattern_local_routing $@ echo -e "Testing different tile organizations"; echo -e "Testing tiles with pins only on top and left sides"; -run-task basic_tests/tile_organization/top_left_custom_pins --debug --show_thread_logs +run-task basic_tests/tile_organization/top_left_custom_pins $@ echo -e "Testing tiles with pins only on top and right sides"; -run-task basic_tests/tile_organization/top_right_custom_pins --debug --show_thread_logs +run-task basic_tests/tile_organization/top_right_custom_pins $@ echo -e "Testing tiles with pins only on bottom and right sides"; -run-task basic_tests/tile_organization/bottom_right_custom_pins --debug --show_thread_logs +run-task basic_tests/tile_organization/bottom_right_custom_pins $@ echo -e "Testing tiles with I/O in center grid"; -run-task basic_tests/tile_organization/tileable_io --debug --show_thread_logs +run-task basic_tests/tile_organization/tileable_io $@ echo -e "Testing global port definition from tiles"; -run-task basic_tests/global_tile_ports/global_tile_clock --debug --show_thread_logs -run-task basic_tests/global_tile_ports/global_tile_reset --debug --show_thread_logs -run-task basic_tests/global_tile_ports/global_tile_4clock --debug --show_thread_logs -run-task basic_tests/global_tile_ports/global_tile_4clock_pin --debug --show_thread_logs +run-task basic_tests/global_tile_ports/global_tile_clock $@ +run-task basic_tests/global_tile_ports/global_tile_reset $@ +run-task basic_tests/global_tile_ports/global_tile_4clock $@ +run-task basic_tests/global_tile_ports/global_tile_4clock_pin $@ echo -e "Testing configuration chain of a K4N4 FPGA using .blif generated by yosys+verific"; -run-task basic_tests/verific_test --debug --show_thread_logs +run-task basic_tests/verific_test $@ echo -e "Testing explicit multi verilog files"; -run-task basic_tests/explicit_multi_verilog_files --debug --show_thread_logs +run-task basic_tests/explicit_multi_verilog_files $@ + +echo -e "Test the remove of runtime directories" +clear-task-run basic_tests/explicit_multi_verilog_files $@ + +echo -e "Testing write GSB to files"; +run-task basic_tests/write_gsb/write_gsb_to_xml $@ +run-task basic_tests/write_gsb/write_gsb_to_xml_compress_routing $@ +run-task basic_tests/write_gsb/write_unique_gsb_to_xml $@ +run-task basic_tests/write_gsb/write_unique_gsb_to_xml_compress_routing $@ + +echo -e "Testing bus group features"; +run-task basic_tests/bus_group/preconfig_testbench_explicit_mapping $@ +run-task basic_tests/bus_group/preconfig_testbench_implicit_mapping $@ +run-task basic_tests/bus_group/full_testbench_explicit_mapping $@ +run-task basic_tests/bus_group/full_testbench_implicit_mapping $@ + +echo -e "Testing fix pins features"; +run-task basic_tests/fix_pins $@ + +echo -e "Testing project templates"; +run-task template_tasks/vpr_blif_template $@ +run-task template_tasks/yosys_vpr_template $@ +run-task template_tasks/vtr_benchmarks_template $@ + +echo -e "Testing create tsk from template and run task" +create-task _task_copy basic_tests/generate_fabric +run-task _task_copy echo -e "Testing output files without time stamp"; -run-task basic_tests/no_time_stamp --debug --show_thread_logs +run-task basic_tests/no_time_stamp $@ # Run git-diff to ensure no changes on the golden netlists git diff --name-status -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' if git diff --name-status --exit-code -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then @@ -159,33 +186,8 @@ else echo -e "Detect changes in golden netlists"; exit 1; fi -echo -e "Test the remove of runtime directories" -run-task basic_tests/explicit_multi_verilog_files --debug --show_thread_logs --remove_run_dir all - # Repgression test to test multi-user enviroment +# Note: Keep this task as the last one!!! cp -r */*/basic_tests/full_testbench/configuration_chain /tmp/ -cd /tmp/ && run-task configuration_chain --debug --show_thread_logs - -echo -e "Testing write GSB to files"; -run-task basic_tests/write_gsb/write_gsb_to_xml --debug --show_thread_logs -run-task basic_tests/write_gsb/write_gsb_to_xml_compress_routing --debug --show_thread_logs -run-task basic_tests/write_gsb/write_unique_gsb_to_xml --debug --show_thread_logs -run-task basic_tests/write_gsb/write_unique_gsb_to_xml_compress_routing --debug --show_thread_logs - -echo -e "Testing bus group features"; -run-task basic_tests/bus_group/preconfig_testbench_explicit_mapping --debug --show_thread_logs -run-task basic_tests/bus_group/preconfig_testbench_implicit_mapping --debug --show_thread_logs -run-task basic_tests/bus_group/full_testbench_explicit_mapping --debug --show_thread_logs -run-task basic_tests/bus_group/full_testbench_implicit_mapping --debug --show_thread_logs - -echo -e "Testing fix pins features"; -run-task basic_tests/fix_pins --debug --show_thread_logs - -echo -e "Testing project templates"; -run-task template_tasks/vpr_blif_template --debug --show_thread_logs -run-task template_tasks/yosys_vpr_template --debug --show_thread_logs -run-task template_tasks/vtr_benchmarks_template --debug --show_thread_logs - -echo -e "Testing create tsk from template and run task" -create-task _task_copy basic_tests/generate_fabric -run-task _task_copy +cd /tmp/ && run-task configuration_chain $@ +cd - diff --git a/openfpga_flow/regression_test_scripts/basic_reg_yosys_only_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_yosys_only_test.sh index 372ceb8f4..1396d27c2 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_yosys_only_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_yosys_only_test.sh @@ -9,8 +9,9 @@ PYTHON_EXEC=python3.8 echo -e "Basic regression tests for Yosys-only flow"; echo -e "Testing configuration chain of a K4N4 FPGA"; -run-task basic_tests/yosys_only --debug --show_thread_logs +run-task basic_tests/yosys_only $@ # Repgression test to test multi-user enviroment cp -r */*/basic_tests/yosys_only /tmp/ -cd /tmp/ && run-task yosys_only --debug --show_thread_logs +cd /tmp/ && run-task yosys_only $@ +cd - diff --git a/openfpga_flow/regression_test_scripts/fpga_bitstream_reg_test.sh b/openfpga_flow/regression_test_scripts/fpga_bitstream_reg_test.sh index 9fa5001b2..b08cf6b51 100755 --- a/openfpga_flow/regression_test_scripts/fpga_bitstream_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/fpga_bitstream_reg_test.sh @@ -9,30 +9,30 @@ PYTHON_EXEC=python3.8 echo -e "FPGA-Bitstream regression tests"; echo -e "Testing bitstream generation for an auto-sized device"; -run-task fpga_bitstream/generate_bitstream/device_auto --debug --show_thread_logs +run-task fpga_bitstream/generate_bitstream/device_auto $@ echo -e "Testing bitstream generation for an 48x48 FPGA device"; -run-task fpga_bitstream/generate_bitstream/device_48x48 --debug --show_thread_logs +run-task fpga_bitstream/generate_bitstream/device_48x48 $@ echo -e "Testing bitstream generation for an 96x96 FPGA device"; -run-task fpga_bitstream/generate_bitstream/device_96x96 --debug --show_thread_logs +run-task fpga_bitstream/generate_bitstream/device_96x96 $@ echo -e "Testing loading architecture bitstream from an external file"; -run-task fpga_bitstream/load_external_architecture_bitstream --debug --show_thread_logs +run-task fpga_bitstream/load_external_architecture_bitstream $@ echo -e "Testing repacker capability in identifying wire LUTs"; -run-task fpga_bitstream/repack_wire_lut --debug --show_thread_logs +run-task fpga_bitstream/repack_wire_lut $@ echo -e "Testing overloading default paths for programmable interconnect when generating bitstream"; -run-task fpga_bitstream/overload_mux_default_path --debug --show_thread_logs +run-task fpga_bitstream/overload_mux_default_path $@ echo -e "Testing outputting I/O mapping result to file"; -run-task fpga_bitstream/write_io_mapping --debug --show_thread_logs +run-task fpga_bitstream/write_io_mapping $@ echo -e "Testing report bitstream distribution to file"; -run-task fpga_bitstream/report_bitstream_distribution/default_depth --debug --show_thread_logs -run-task fpga_bitstream/report_bitstream_distribution/custom_depth --debug --show_thread_logs +run-task fpga_bitstream/report_bitstream_distribution/default_depth $@ +run-task fpga_bitstream/report_bitstream_distribution/custom_depth $@ echo -e "Testing bitstream file with don't care bits"; -run-task fpga_bitstream/dont_care_bits/ql_memory_bank_flatten --debug --show_thread_logs -run-task fpga_bitstream/dont_care_bits/ql_memory_bank_shift_register --debug --show_thread_logs +run-task fpga_bitstream/dont_care_bits/ql_memory_bank_flatten $@ +run-task fpga_bitstream/dont_care_bits/ql_memory_bank_shift_register $@ diff --git a/openfpga_flow/regression_test_scripts/fpga_sdc_reg_test.sh b/openfpga_flow/regression_test_scripts/fpga_sdc_reg_test.sh index 713c328b9..084b0ea20 100755 --- a/openfpga_flow/regression_test_scripts/fpga_sdc_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/fpga_sdc_reg_test.sh @@ -9,12 +9,12 @@ PYTHON_EXEC=python3.8 echo -e "FPGA-SDC regression tests"; echo -e "Testing SDC generation with time units"; -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_as --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_fs --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ps --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ns --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_us --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ms --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_default --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ks --debug --show_thread_logs -run-task fpga_sdc/sdc_time_unit/sdc_time_unit_Ms --debug --show_thread_logs +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_as $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_fs $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ps $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ns $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_us $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ms $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_default $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_ks $@ +run-task fpga_sdc/sdc_time_unit/sdc_time_unit_Ms $@ diff --git a/openfpga_flow/regression_test_scripts/fpga_spice_reg_test.sh b/openfpga_flow/regression_test_scripts/fpga_spice_reg_test.sh index b58d75f8e..c003f5d93 100755 --- a/openfpga_flow/regression_test_scripts/fpga_spice_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/fpga_spice_reg_test.sh @@ -9,4 +9,4 @@ PYTHON_EXEC=python3.8 echo -e "FPGA-SPICE regression tests"; echo -e "Testing FPGA-SPICE with netlist generation"; -run-task fpga_spice/generate_spice --debug --show_thread_logs +run-task fpga_spice/generate_spice $@ diff --git a/openfpga_flow/regression_test_scripts/fpga_verilog_reg_test.sh b/openfpga_flow/regression_test_scripts/fpga_verilog_reg_test.sh index aea6a3459..3cad92b31 100755 --- a/openfpga_flow/regression_test_scripts/fpga_verilog_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/fpga_verilog_reg_test.sh @@ -9,141 +9,141 @@ PYTHON_EXEC=python3.8 echo -e "FPGA-Verilog Feature Tests"; echo -e "Testing Verilog generation for LUTs: a single mode LUT6 FPGA using micro benchmarks"; -run-task fpga_verilog/lut_design/single_mode --debug --show_thread_logs +run-task fpga_verilog/lut_design/single_mode $@ echo -e "Testing Verilog generation for LUTs: simple fracturable LUT4 "; -run-task fpga_verilog/lut_design/frac_lut4 --debug --show_thread_logs +run-task fpga_verilog/lut_design/frac_lut4 $@ echo -e "Testing Verilog generation for LUTs: fracturable LUT4 with embedded carry logic"; -run-task fpga_verilog/lut_design/frac_lut4_arith --debug --show_thread_logs +run-task fpga_verilog/lut_design/frac_lut4_arith $@ echo -e "Testing Verilog generation for LUTs: native fracturable LUT4 "; -run-task fpga_verilog/lut_design/frac_native_lut4 --debug --show_thread_logs +run-task fpga_verilog/lut_design/frac_native_lut4 $@ echo -e "Testing Verilog generation for LUTs: simple fracturable LUT4 using AND gate to switch modes"; -run-task fpga_verilog/lut_design/frac_lut4_and_switch --debug --show_thread_logs +run-task fpga_verilog/lut_design/frac_lut4_and_switch $@ echo -e "Testing Verilog generation for LUTs: simple fracturable LUT6 "; -run-task fpga_verilog/lut_design/frac_lut6 --debug --show_thread_logs +run-task fpga_verilog/lut_design/frac_lut6 $@ echo -e "Testing Verilog generation for LUTs: LUT6 with intermediate buffers"; -run-task fpga_verilog/lut_design/intermediate_buffer --debug --show_thread_logs +run-task fpga_verilog/lut_design/intermediate_buffer $@ echo -e "Testing Verilog generation with VPR's untileable routing architecture "; -run-task fpga_verilog/untileable --debug --show_thread_logs +run-task fpga_verilog/untileable $@ echo -e "Testing Verilog generation with hard adder chain in CLBs "; -run-task fpga_verilog/adder/hard_adder --debug --show_thread_logs +run-task fpga_verilog/adder/hard_adder $@ echo -e "Testing Verilog generation with soft adder chain in CLBs "; -run-task fpga_verilog/adder/soft_adder --debug --show_thread_logs +run-task fpga_verilog/adder/soft_adder $@ echo -e "Testing Verilog generation with 1k block RAMs "; -run-task fpga_verilog/bram/dpram1k --debug --show_thread_logs +run-task fpga_verilog/bram/dpram1k $@ echo -e "Testing Verilog generation with 1k block RAMs spanning two columns "; -run-task fpga_verilog/bram/wide_dpram1k --debug --show_thread_logs +run-task fpga_verilog/bram/wide_dpram1k $@ echo -e "Testing Verilog generation with heterogeneous fabric using 8-bit single-mode multipliers "; -run-task fpga_verilog/dsp/single_mode_mult_8x8 --debug --show_thread_logs +run-task fpga_verilog/dsp/single_mode_mult_8x8 $@ echo -e "Testing Verilog generation with heterogeneous fabric using 16-bit multi-mode multipliers "; -run-task fpga_verilog/dsp/multi_mode_mult_16x16 --debug --show_thread_logs +run-task fpga_verilog/dsp/multi_mode_mult_16x16 $@ echo -e "Testing Verilog generation with heterogeneous fabric using multi-width 16-bit multi-mode multipliers "; -run-task fpga_verilog/dsp/wide_multi_mode_mult_16x16 --debug --show_thread_logs +run-task fpga_verilog/dsp/wide_multi_mode_mult_16x16 $@ echo -e "Testing Verilog generation with heterogeneous fabric using 8-bit single-mode registerable multipliers "; -run-task fpga_verilog/dsp/single_mode_mult_8x8_reg --debug --show_thread_logs +run-task fpga_verilog/dsp/single_mode_mult_8x8_reg $@ echo -e "Testing Verilog generation with different I/O capacities on each side of an FPGA "; -run-task fpga_verilog/io/multi_io_capacity --debug --show_thread_logs +run-task fpga_verilog/io/multi_io_capacity $@ echo -e "Testing Verilog generation with I/Os only on left and right sides of an FPGA "; -run-task fpga_verilog/io/reduced_io --debug --show_thread_logs +run-task fpga_verilog/io/reduced_io $@ echo -e "Testing Verilog generation with embedded I/Os for an FPGA "; -run-task fpga_verilog/io/embedded_io --debug --show_thread_logs +run-task fpga_verilog/io/embedded_io $@ echo -e "Testing Verilog generation with SoC I/Os for an FPGA "; -run-task fpga_verilog/io/soc_io --debug --show_thread_logs +run-task fpga_verilog/io/soc_io $@ echo -e "Testing Verilog generation with registerable I/Os for an FPGA "; -run-task fpga_verilog/io/registerable_io --debug --show_thread_logs +run-task fpga_verilog/io/registerable_io $@ echo -e "Testing Verilog generation with adder chain across an FPGA"; -run-task fpga_verilog/fabric_chain/adder_chain --debug --show_thread_logs +run-task fpga_verilog/fabric_chain/adder_chain $@ echo -e "Testing Verilog generation with shift register chain across an FPGA"; -run-task fpga_verilog/fabric_chain/register_chain --debug --show_thread_logs +run-task fpga_verilog/fabric_chain/register_chain $@ echo -e "Testing Verilog generation with scan chain across an FPGA"; -run-task fpga_verilog/fabric_chain/scan_chain --debug --show_thread_logs +run-task fpga_verilog/fabric_chain/scan_chain $@ echo -e "Testing Verilog generation with routing multiplexers implemented by tree structure"; -run-task fpga_verilog/mux_design/tree_structure --debug --show_thread_logs +run-task fpga_verilog/mux_design/tree_structure $@ echo -e "Testing Verilog generation with routing multiplexers implemented by standard cell MUX2"; -run-task fpga_verilog/mux_design/stdcell_mux2 --debug --show_thread_logs +run-task fpga_verilog/mux_design/stdcell_mux2 $@ echo -e "Testing Verilog generation with routing multiplexers implemented by local encoders"; -run-task fpga_verilog/mux_design/local_encoder --debug --show_thread_logs +run-task fpga_verilog/mux_design/local_encoder $@ echo -e "Testing Verilog generation with routing multiplexers without buffers"; -run-task fpga_verilog/mux_design/debuf_mux --debug --show_thread_logs +run-task fpga_verilog/mux_design/debuf_mux $@ echo -e "Testing Verilog generation with routing multiplexers with input buffers only"; -run-task fpga_verilog/mux_design/inbuf_only_mux --debug --show_thread_logs +run-task fpga_verilog/mux_design/inbuf_only_mux $@ echo -e "Testing Verilog generation with routing multiplexers with output buffers only"; -run-task fpga_verilog/mux_design/outbuf_only_mux --debug --show_thread_logs +run-task fpga_verilog/mux_design/outbuf_only_mux $@ echo -e "Testing Verilog generation with routing multiplexers with constant gnd input"; -run-task fpga_verilog/mux_design/const_input_gnd --debug --show_thread_logs +run-task fpga_verilog/mux_design/const_input_gnd $@ echo -e "Testing Verilog generation with routing multiplexers without constant inputs"; -run-task fpga_verilog/mux_design/no_const_input --debug --show_thread_logs +run-task fpga_verilog/mux_design/no_const_input $@ echo -e "Testing Verilog generation with behavioral description"; -run-task fpga_verilog/verilog_netlist_formats/behavioral_verilog --debug --show_thread_logs -run-task fpga_verilog/verilog_netlist_formats/behavioral_verilog_default_nettype_wire --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/behavioral_verilog $@ +run-task fpga_verilog/verilog_netlist_formats/behavioral_verilog_default_nettype_wire $@ echo -e "Testing synthesizable Verilog generation with external standard cells"; -run-task fpga_verilog/verilog_netlist_formats/synthesizable_verilog --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/synthesizable_verilog $@ echo -e "Testing implicit Verilog generation"; -run-task fpga_verilog/verilog_netlist_formats/implicit_verilog --debug --show_thread_logs -run-task fpga_verilog/verilog_netlist_formats/implicit_verilog_default_nettype_wire --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/implicit_verilog $@ +run-task fpga_verilog/verilog_netlist_formats/implicit_verilog_default_nettype_wire $@ echo -e "Testing explicit Verilog generation"; -run-task fpga_verilog/verilog_netlist_formats/explicit_port_mapping_default_nettype_wire --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/explicit_port_mapping_default_nettype_wire $@ echo -e "Testing Verilog generation with flatten routing modules"; -run-task fpga_verilog/flatten_routing --debug --show_thread_logs +run-task fpga_verilog/flatten_routing $@ echo -e "Testing Verilog generation with duplicated grid output pins"; -run-task fpga_verilog/duplicated_grid_pin --debug --show_thread_logs +run-task fpga_verilog/duplicated_grid_pin $@ echo -e "Testing Verilog generation with spy output pads"; -run-task fpga_verilog/spypad --debug --show_thread_logs +run-task fpga_verilog/spypad $@ echo -e "Testing Power-gating designs"; run-task fpga_verilog/power_gated_design/power_gated_inverter --show_thread_logs --debug echo -e "Testing Depopulated crossbar in local routing"; -run-task fpga_verilog/depopulate_crossbar --debug --show_thread_logs +run-task fpga_verilog/depopulate_crossbar $@ echo -e "Testing Fully connected output crossbar in local routing"; -run-task fpga_verilog/fully_connected_output_crossbar --debug --show_thread_logs +run-task fpga_verilog/fully_connected_output_crossbar $@ echo -e "Testing through channels in tileable routing"; -run-task fpga_verilog/thru_channel/thru_narrow_tile --debug --show_thread_logs -run-task fpga_verilog/thru_channel/thru_wide_tile --debug --show_thread_logs +run-task fpga_verilog/thru_channel/thru_narrow_tile $@ +run-task fpga_verilog/thru_channel/thru_wide_tile $@ echo -e "Testing the generation of preconfigured fabric wrapper for different HDL simulators"; -run-task fpga_verilog/verilog_netlist_formats/embed_bitstream_none --debug --show_thread_logs -run-task fpga_verilog/verilog_netlist_formats/embed_bitstream_modelsim --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/embed_bitstream_none $@ +run-task fpga_verilog/verilog_netlist_formats/embed_bitstream_modelsim $@ echo -e "Testing the netlist generation by forcing the use of relative paths"; -run-task fpga_verilog/verilog_netlist_formats/use_relative_path --debug --show_thread_logs -run-task fpga_verilog/verilog_netlist_formats/preconfig_testbench_use_relative_path --debug --show_thread_logs +run-task fpga_verilog/verilog_netlist_formats/use_relative_path $@ +run-task fpga_verilog/verilog_netlist_formats/preconfig_testbench_use_relative_path $@ diff --git a/openfpga_flow/regression_test_scripts/iwls_benchmark_reg_test.sh b/openfpga_flow/regression_test_scripts/iwls_benchmark_reg_test.sh index 1c00bd150..4b37f6da7 100755 --- a/openfpga_flow/regression_test_scripts/iwls_benchmark_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/iwls_benchmark_reg_test.sh @@ -7,7 +7,7 @@ PYTHON_EXEC=python3.8 # OpenFPGA Shell with VPR8 ############################################## echo -e "IWLS'05 benchmark regression tests"; -run-task benchmark_sweep/iwls2005 --debug --show_thread_logs +run-task benchmark_sweep/iwls2005 $@ # Run a quick but relaxed QoR check for heterogeneous blocks #python3 openfpga_flow/scripts/check_qor.py --reference_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/config/vtr_benchmark_golden_results.csv --check_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/latest/task_result.csv --metric_checklist_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/config/metric_checklist.csv --check_tolerance 0.2,100 python3 openfpga_flow/scripts/check_qor.py --reference_csv_file openfpga_flow/tasks/benchmark_sweep/iwls2005/config/iwls_benchmark_golden_results.csv --check_csv_file openfpga_flow/tasks/benchmark_sweep/iwls2005/latest/task_result.csv --metric_checklist_csv_file openfpga_flow/tasks/benchmark_sweep/iwls2005/config/metric_checklist.csv --check_tolerance 0.2,100 diff --git a/openfpga_flow/regression_test_scripts/micro_benchmark_reg_test.sh b/openfpga_flow/regression_test_scripts/micro_benchmark_reg_test.sh index 134b5f4f4..cb7b41252 100755 --- a/openfpga_flow/regression_test_scripts/micro_benchmark_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/micro_benchmark_reg_test.sh @@ -7,15 +7,15 @@ PYTHON_EXEC=python3.8 # OpenFPGA Shell with VPR8 ############################################## echo -e "Micro benchmark regression tests"; -run-task benchmark_sweep/counter8 --debug --show_thread_logs -run-task benchmark_sweep/counter8_full_testbench --debug --show_thread_logs -run-task benchmark_sweep/counter128 --debug --show_thread_logs -run-task benchmark_sweep/mac_units --debug --show_thread_logs +run-task benchmark_sweep/counter8 $@ +run-task benchmark_sweep/counter8_full_testbench $@ +run-task benchmark_sweep/counter128 $@ +run-task benchmark_sweep/mac_units $@ # Verify MCNC big20 benchmark suite with ModelSim # Please make sure you have ModelSim installed in the environment # Otherwise, it will fail -run-task benchmark_sweep/mcnc_big20 --debug --show_thread_logs +run-task benchmark_sweep/mcnc_big20 $@ #python3 openfpga_flow/scripts/run_modelsim.py mcnc_big20 --run_sim -run-task benchmark_sweep/signal_gen --debug --show_thread_logs +run-task benchmark_sweep/signal_gen $@ diff --git a/openfpga_flow/regression_test_scripts/quicklogic_reg_test.sh b/openfpga_flow/regression_test_scripts/quicklogic_reg_test.sh index 2d3bd1202..2279a5c86 100755 --- a/openfpga_flow/regression_test_scripts/quicklogic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/quicklogic_reg_test.sh @@ -9,11 +9,11 @@ PYTHON_EXEC=python3.8 echo -e "QuickLogic regression tests"; echo -e "Testing yosys flow using custom ys script for running quicklogic device"; -run-task quicklogic_tests/flow_test --debug --show_thread_logs +run-task quicklogic_tests/flow_test $@ echo -e "Testing yosys flow using custom ys script for running multi-clock quicklogic device"; -run-task quicklogic_tests/counter_5clock_test --debug --show_thread_logs -run-task quicklogic_tests/sdc_controller_test --debug --show_thread_logs +run-task quicklogic_tests/counter_5clock_test $@ +run-task quicklogic_tests/sdc_controller_test $@ echo -e "Testing yosys flow using custom ys script for adders in quicklogic device"; -run-task quicklogic_tests/lut_adder_test --debug --show_thread_logs +run-task quicklogic_tests/lut_adder_test $@ diff --git a/openfpga_flow/regression_test_scripts/vtr_benchmark_reg_test.sh b/openfpga_flow/regression_test_scripts/vtr_benchmark_reg_test.sh index 4ca761a3e..8bbc32436 100755 --- a/openfpga_flow/regression_test_scripts/vtr_benchmark_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/vtr_benchmark_reg_test.sh @@ -7,6 +7,6 @@ PYTHON_EXEC=python3.8 # OpenFPGA Shell with VPR8 ############################################## echo -e "VTR benchmark regression tests"; -run-task benchmark_sweep/vtr_benchmarks --debug --show_thread_logs +run-task benchmark_sweep/vtr_benchmarks $@ # Run a quick but relaxed QoR check for heterogeneous blocks python3 openfpga_flow/scripts/check_qor.py --reference_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/config/vtr_benchmark_golden_results.csv --check_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/latest/task_result.csv --metric_checklist_csv_file openfpga_flow/tasks/benchmark_sweep/vtr_benchmarks/config/metric_checklist.csv --check_tolerance 0.2,100 From e56775cd00b4c57ebf46efc2226feea01b1d8ba2 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:18:07 +0800 Subject: [PATCH 04/13] [ci] force debug run for regression tests --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b2d65cba..985256533 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -264,7 +264,7 @@ jobs: chmod +x yosys/install/bin/yosys-smtbmc - name: ${{matrix.config.name}}_GCC-8_(Ubuntu 18.04) shell: bash - run: source openfpga.sh && source openfpga_flow/regression_test_scripts/${{matrix.config.name}}.sh + run: source openfpga.sh && source openfpga_flow/regression_test_scripts/${{matrix.config.name}}.sh --debug --show_thread_logs - name: Upload artifact uses: actions/upload-artifact@v2 if: ${{ failure() }} @@ -310,7 +310,7 @@ jobs: ${PYTHON_EXEC} -m pip install -r requirements.txt rsync -am --exclude='openfpga_flow/**' /opt/openfpga/. . unset OPENFPGA_PATH - source openfpga.sh && source openfpga_flow/regression_test_scripts/${{matrix.config.name}}.sh + source openfpga.sh && source openfpga_flow/regression_test_scripts/${{matrix.config.name}}.sh --debug --show_thread_logs - name: Upload artifact uses: actions/upload-artifact@v2 if: ${{ failure() }} From e4c1097fdb38ffa0f589afe9e47e149642527f31 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:28:45 +0800 Subject: [PATCH 05/13] [test] typo --- openfpga.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfpga.sh b/openfpga.sh index e31171127..10dbdcf17 100755 --- a/openfpga.sh +++ b/openfpga.sh @@ -65,7 +65,7 @@ clean-run () { rm -rf ./openfpga_flow/**/run??? } -run-task-run () { +clear-task-run () { $PYTHON_EXEC $OPENFPGA_SCRIPT_PATH/run_fpga_task.py --remove_run_dir all } From 61a1462c21e1da2ed040a40b9af375b6a7f9f0ec Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:44:48 +0800 Subject: [PATCH 06/13] [doc] add guidelines about running regression tests --- docs/source/dev_manual/index.rst | 2 + docs/source/dev_manual/regression_tests.rst | 46 +++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 docs/source/dev_manual/regression_tests.rst diff --git a/docs/source/dev_manual/index.rst b/docs/source/dev_manual/index.rst index 6e7acec12..df2d72867 100644 --- a/docs/source/dev_manual/index.rst +++ b/docs/source/dev_manual/index.rst @@ -5,3 +5,5 @@ ci_cd_setup/index version_number + + regression_tests diff --git a/docs/source/dev_manual/regression_tests.rst b/docs/source/dev_manual/regression_tests.rst new file mode 100644 index 000000000..90c45babf --- /dev/null +++ b/docs/source/dev_manual/regression_tests.rst @@ -0,0 +1,46 @@ +.. dev_manual_regression_tests:: + +Regression Tests +================ + +Regression tests are designed to cover various technical features of the OpenFPGA projects, including but not limited to + +- Netlist generation +- Netlist verification +- Bitstream generation + +Considering the large number of technical features, regression tests are categorized into several groups, which can be found at ``openfpga_flow/regression_test_scripts/`` + +Run a Test +---------- + +To run a regression test, users can execute a shell script (assume you are under the root directory of the project), for example, + +``` +./openfpga_flow/regression_test_scripts/basic_reg_test.sh [OPTIONS] +``` + +.. note:: ``basic_reg_test`` can be replaced by other tests which are under ``openfpga_flow/regression_test_scripts/`` + +Test Options +------------ + +There are a few options available when running the tests. + +.. option:: --debug + + This option can turn on debug mode when running regression tests. By default it is ``off``. + +.. option:: --show_thread_logs + + This option can enable verbose output when running regression tests. By default it is ``off``. + +.. note:: To avoid massive outputs, suggest to run the tests with default options. In CI, always recommend to turn on the debug and verbose options + +.. option:: --remove_run_dir all + + This option is to remove all the previous run results for a specific regression test. Suggest to use when there are limited disk space. + + .. note:: Be careful before using this option! It may cause permanent loss on test results. + + From 07df4611e82eab1e39cbf4d6dd15ab7c9f000e55 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 13:46:13 +0800 Subject: [PATCH 07/13] [doc] more tips --- docs/source/dev_manual/regression_tests.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/dev_manual/regression_tests.rst b/docs/source/dev_manual/regression_tests.rst index 90c45babf..9dc26b3f5 100644 --- a/docs/source/dev_manual/regression_tests.rst +++ b/docs/source/dev_manual/regression_tests.rst @@ -14,6 +14,8 @@ Considering the large number of technical features, regression tests are categor Run a Test ---------- +.. note:: Make sure you have compiled OpenFPGA and set up your environment before reaching this step. See details in :ref:`getting_started_tutorials`. + To run a regression test, users can execute a shell script (assume you are under the root directory of the project), for example, ``` From 1794578b497672b9158677e9ce3bc53bbcff6b6e Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 14:04:12 +0800 Subject: [PATCH 08/13] [doc] code format --- docs/source/dev_manual/regression_tests.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/dev_manual/regression_tests.rst b/docs/source/dev_manual/regression_tests.rst index 9dc26b3f5..89ebaf7c8 100644 --- a/docs/source/dev_manual/regression_tests.rst +++ b/docs/source/dev_manual/regression_tests.rst @@ -18,9 +18,9 @@ Run a Test To run a regression test, users can execute a shell script (assume you are under the root directory of the project), for example, -``` -./openfpga_flow/regression_test_scripts/basic_reg_test.sh [OPTIONS] -``` +.. code-block:: shell + + ./openfpga_flow/regression_test_scripts/basic_reg_test.sh [OPTIONS] .. note:: ``basic_reg_test`` can be replaced by other tests which are under ``openfpga_flow/regression_test_scripts/`` From 03fc790395d5836c0f94a06da2917e2790843004 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Sun, 22 May 2022 14:47:25 +0800 Subject: [PATCH 09/13] [test] bug fix --- openfpga.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfpga.sh b/openfpga.sh index 10dbdcf17..81a6279ad 100755 --- a/openfpga.sh +++ b/openfpga.sh @@ -66,7 +66,7 @@ clean-run () { } clear-task-run () { - $PYTHON_EXEC $OPENFPGA_SCRIPT_PATH/run_fpga_task.py --remove_run_dir all + $PYTHON_EXEC $OPENFPGA_SCRIPT_PATH/run_fpga_task.py "$@" --remove_run_dir all } run-modelsim () { From 488a93409758d8d554077c059517e9a30b4e095e Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 23 May 2022 09:12:33 +0800 Subject: [PATCH 10/13] [test] give abs path for git diff in basic regression tests --- openfpga_flow/regression_test_scripts/basic_reg_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfpga_flow/regression_test_scripts/basic_reg_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_test.sh index 0134b21e2..2fe51c168 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_test.sh @@ -179,8 +179,8 @@ run-task _task_copy echo -e "Testing output files without time stamp"; run-task basic_tests/no_time_stamp $@ # Run git-diff to ensure no changes on the golden netlists -git diff --name-status -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' -if git diff --name-status --exit-code -- ':openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then +git diff --name-status -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' +if git diff --name-status --exit-code -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then echo -e "Golden netlist remain unchanged" else echo -e "Detect changes in golden netlists"; exit 1; From b41cbad5d3da68291c56694c11eda249d4c385f8 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 23 May 2022 10:32:43 +0800 Subject: [PATCH 11/13] [test] force to run git diff under root directory --- openfpga_flow/regression_test_scripts/basic_reg_test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openfpga_flow/regression_test_scripts/basic_reg_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_test.sh index 2fe51c168..9b2ecebc7 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_test.sh @@ -179,12 +179,15 @@ run-task _task_copy echo -e "Testing output files without time stamp"; run-task basic_tests/no_time_stamp $@ # Run git-diff to ensure no changes on the golden netlists +# Switch to root path in case users are running the tests in another location +cd ${OPENFPGA_PATH} git diff --name-status -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' if git diff --name-status --exit-code -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then echo -e "Golden netlist remain unchanged" else echo -e "Detect changes in golden netlists"; exit 1; fi +cd - # Repgression test to test multi-user enviroment # Note: Keep this task as the last one!!! From 7d694acf32d8a63a412c005a2bd27c0a46fc51bf Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 23 May 2022 11:21:36 +0800 Subject: [PATCH 12/13] [test] debugging basic reg test paths --- openfpga_flow/regression_test_scripts/basic_reg_test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openfpga_flow/regression_test_scripts/basic_reg_test.sh b/openfpga_flow/regression_test_scripts/basic_reg_test.sh index 9b2ecebc7..9572da2d5 100755 --- a/openfpga_flow/regression_test_scripts/basic_reg_test.sh +++ b/openfpga_flow/regression_test_scripts/basic_reg_test.sh @@ -181,6 +181,9 @@ run-task basic_tests/no_time_stamp $@ # Run git-diff to ensure no changes on the golden netlists # Switch to root path in case users are running the tests in another location cd ${OPENFPGA_PATH} +pwd +git config --global --add safe.directory ${OPENFPGA_PATH} +git log git diff --name-status -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**' if git diff --name-status --exit-code -- ':${OPENFPGA_PATH}/openfpga_flow/tasks/basic_tests/no_time_stamp/golden_outputs_no_time_stamp/**'; then echo -e "Golden netlist remain unchanged" From c53c058d1eab2ca4fc66aa3cdce5b66e21b5e314 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 23 May 2022 12:16:52 +0800 Subject: [PATCH 13/13] [docker] add safe directory which can allow git diff --- docker/Dockerfile.master | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.master b/docker/Dockerfile.master index 55a46fa47..686d0820c 100644 --- a/docker/Dockerfile.master +++ b/docker/Dockerfile.master @@ -22,6 +22,7 @@ RUN adduser --disabled-password \ USER root RUN chown -R ${NB_UID} ${HOME} RUN chmod -R +r /opt/openfpga/ +git config --global --add safe.directory * USER ${NB_USER} WORKDIR ${HOME}