Merge pull request #1490 from lnis-uofu/xt_flow

Update openfpga flow to allow custom default tool path configuration

docs/source/manual/openfpga_flow/run_fpga_task.rst

@@ -29,9 +29,9 @@ Similarly  ``regression/regression_quick`` expect following structure::
 Running OpenFPGA Task:
 ~~~~~~~~~~~~~~~~~~~~~~
 
-At a minimum ``open_fpga_flow.py`` requires following command-line arguments::
+At a minimum ``run_fpga_task.py`` requires following command-line arguments::
 
-    open_fpga_flow.py <task1_name> <task2_name> ... [<options>]
+    run_fpga_task.py <task1_name> <task2_name> ... [<options>]
 
 where:
 
@@ -58,6 +58,12 @@ Command-line Options
     if any threads fail to execute successfully. It is mainly used to while
     performing regression test.
 
+.. option:: --default_tool_path
+
+    Specify the paths to tools as well as the keywords to extract QoR results from log files, when running this task. By default, the script will use the ``openfpga_flow/misc/fpgaflow_default_tool_path.conf``.
+
+    .. note:: Please use absolute path!!!
+
 .. option:: --test_run
 
     This option allows to debug OpenFPGA Task script

openfpga_flow/misc/fpgaflow_default_tool_path.conf

@@ -22,8 +22,6 @@ clb_blocks = "Netlist clb blocks: ([0-9]+)", str
 io_blocks = "Netlist io blocks: ([0-9]+)", str
 mult_blocks = "Netlist mult_36 blocks: ([0-9]+)", str
 memory_blocks = "Netlist memory blocks: ([0-9]+)", str
-logic_delay = "Total logic delay: ([0-9.]+)", str
-total_net_delay = "total net delay: ([0-9.]+)", str
 total_routing_area = "Total routing area: ([0-9.]+[e|E\+[0-9]+)", str
 total_logic_block_area = "Total used logic block area: ([0-9.]+[e|E\+[0-9]+)", str
 total_wire_length = "Total wirelength: ([0-9]+)", str
@@ -31,7 +29,7 @@ packing_time = "Packing took ([0-9.]+) seconds", str
 placement_time = "Placement took ([0-9.]+) seconds", str
 routing_time = "Routing took ([0-9.]+) seconds", str
 average_net_length = "average net length: ([0-9.]+)", str
-critical_path = "Final critical path: ([0-9.]+) ([a-z])s", scientific
+critical_path = "Final critical path delay \(least slack\): ([0-9.]+) ([a-z])s", scientific
 total_routing_time = "Routing took ([0-9.]+) seconds", float
 
 [DEFAULT_PARSE_RESULT_POWER]

openfpga_flow/misc/fpgaflow_default_tool_path_timing.conf

@@ -0,0 +1,76 @@
+# Standard Configuration Example
+[CAD_TOOLS_PATH]
+openfpga_shell_path = ${PATH:OPENFPGA_PATH}/build/openfpga/openfpga
+yosys_path = ${PATH:OPENFPGA_PATH}/build/yosys/bin/yosys
+misc_dir = ${PATH:OPENFPGA_PATH}/openfpga_flow/misc
+odin2_path = ${PATH:OPENFPGA_PATH}/openfpga_flow/not_used_atm/odin2.exe
+abc_path = ${PATH:OPENFPGA_PATH}/build/yosys/bin/yosys-abc
+abc_mccl_path = ${PATH:OPENFPGA_PATH}/build/vtr-verilog-to-routing/abc/abc
+abc_with_bb_support_path = ${PATH:OPENFPGA_PATH}/build/vtr-verilog-to-routing/abc/abc
+vpr_path = ${PATH:OPENFPGA_PATH}/build/vtr-verilog-to-routing/vpr/vpr
+ace_path = ${PATH:OPENFPGA_PATH}/build/vtr-verilog-to-routing/ace2/ace
+pro_blif_path = ${PATH:OPENFPGA_PATH}/openfpga_flow/scripts/pro_blif.pl
+iverilog_path = iverilog
+include_netlist_verification = ${PATH:OPENFPGA_PATH}/vpr/VerilogNetlists
+
+[FLOW_SCRIPT_CONFIG]
+valid_flows = vpr_blif,yosys_vpr
+
+[DEFAULT_PARSE_RESULT_VPR]
+# parser format <name of variable> = <regex string>, <lambda function/type>
+clb_blocks = "Netlist clb blocks: ([0-9]+)", str
+io_blocks = "Netlist io blocks: ([0-9]+)", str
+mult_blocks = "Netlist mult_36 blocks: ([0-9]+)", str
+memory_blocks = "Netlist memory blocks: ([0-9]+)", str
+total_routing_area = "Total routing area: ([0-9.]+[e|E\+[0-9]+)", str
+total_logic_block_area = "Total used logic block area: ([0-9.]+[e|E\+[0-9]+)", str
+total_wire_length = "Total wirelength: ([0-9]+)", str
+packing_time = "Packing took ([0-9.]+) seconds", str
+placement_time = "Placement took ([0-9.]+) seconds", str
+routing_time = "Routing took ([0-9.]+) seconds", str
+average_net_length = "average net length: ([0-9.]+)", str
+critical_path = "Final critical path delay \(least slack\): ([0-9.]+) ([a-z])s", scientific
+vclk0_critical_path = "virtual_io_clock to clk0 CPD: ([0-9.]+) ([a-z])s", scientific
+clk0_critical_path = "clk0 to clk0 CPD: ([0-9.]+) ([a-z])s", scientific
+vclk1_critical_path = "virtual_io_clock to clk1 CPD: ([0-9.]+) ([a-z])s", scientific
+clk1_critical_path = "clk1 to clk1 CPD: ([0-9.]+) ([a-z])s", scientific
+total_routing_time = "Routing took ([0-9.]+) seconds", float
+
+[DEFAULT_PARSE_RESULT_POWER]
+pb_type_power="PB Types\s+([0-9]+)", str
+routing_power="Routing\s+([0-9]+)", str
+switch_box_power="Switch Box\s+([0-9]+)", str
+connection_box_power="Connection Box\s+([0-9]+)", str
+primitives_power="Primitives\s+([0-9]+)", str
+interc_structures_power="Interc Structures\s+([0-9]+)", str
+lut6_power="^\s+lut6\s+([0-9]+)", str
+ff_power="^\s+ff\s+([0-9]+)", str
+
+[INTERMIDIATE_FILE_PREFIX]
+# Yosys files
+yosys_out_blif=${PATH:TOP_MODULE}_yosys_out.blif
+yosys_output=yosys_output.txt
+
+# ACE2 and intermidiate file
+activity_file=${PATH:TOP_MODULE}_ace_out.act
+ace_output_blif=${PATH:TOP_MODULE}_ace_out.blif
+corrected_format_blif=${PATH:TOP_MODULE}.blif
+blackbox_blif=${PATH:TOP_MODULE}_bb.blif
+
+# VPR Files
+min_chann_vpr_output=${PATH:TOP_MODULE}_min_chan_width_vpr.txt
+reroute_chan_vpr_output=${PATH:TOP_MODULE}_reroute_vpr.txt
+fixed_chan_vpr_output=${PATH:TOP_MODULE}_fr_chan_width.txt
+vpr_stat_parse_fn=vpr_stat.txt
+vpr_power_stat_parse_fn=vpr_power_stat.txt
+vpr_net_file=${PATH:TOP_MODULE}_vpr.net
+vpr_place_file=${PATH:TOP_MODULE}_vpr.place
+vpr_route_file=${PATH:TOP_MODULE}_vpr.route
+
+#Iverilog verification file
+iverilog_output=iverilog_output.txt
+vvp_output=vvp_sim_output.txt
+
+[CMD_ARGUMENT_DEPENDANCY]
+vpr_fpga_verilog=vpr_fpga_verilog_dir|abc
+vpr_fpga_verilog_dir=vpr_fpga_verilog

openfpga_flow/regression_test_scripts/basic_reg_test.sh

@@ -218,7 +218,7 @@ run-task basic_tests/global_tile_ports/global_tile_clock_subtile $@
 run-task basic_tests/global_tile_ports/global_tile_clock_subtile_port_merge $@
 run-task basic_tests/global_tile_ports/global_tile_clock_subtile_port_merge_fabric_tile_group_config $@
 run-task basic_tests/global_tile_ports/global_tile_reset $@
-run-task basic_tests/global_tile_ports/global_tile_4clock $@
+run-task basic_tests/global_tile_ports/global_tile_4clock --default_tool_path ${OPENFPGA_PATH}/openfpga_flow/misc/fpgaflow_default_tool_path_timing.conf $@
 run-task basic_tests/global_tile_ports/global_tile_4clock_pin $@
 
 echo -e "Testing programmable clock architecture";

openfpga_flow/scripts/run_fpga_flow.py

@@ -89,6 +89,12 @@ parser.add_argument(
     default=os.path.join(openfpga_base_dir, "tmp"),
     help="Directory to store intermidiate file & final results",
 )
+parser.add_argument(
+    "--default_tool_path",
+    type=str,
+    default=os.path.join(flow_script_dir, os.pardir, "misc", "fpgaflow_default_tool_path.conf"),
+    help="The configuraton file contains paths to tools as well as keywords to be extracted from logs",
+)
 parser.add_argument(
     "--openfpga_shell_template",
     type=str,
@@ -332,8 +338,8 @@ ExecTime = {}
 
 def main():
     logger.debug("Script Launched in " + os.getcwd())
-    check_required_file()
+    check_required_file(args.default_tool_path)
-    read_script_config()
+    read_script_config(args.default_tool_path)
     validate_command_line_arguments()
     prepare_run_directory(args.run_dir)
     if args.fpga_flow == "yosys_vpr":
@@ -394,26 +400,22 @@ def main():
 # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
 
 
-def check_required_file():
+def check_required_file(default_tool_path):
     """Function ensure existace of all required files for the script"""
     files_dict = {
-        "CAD TOOL PATH": os.path.join(
+        "CAD TOOL PATH": default_tool_path,
-            flow_script_dir, os.pardir, "misc", "fpgaflow_default_tool_path.conf"
-        ),
     }
     for filename, filepath in files_dict.items():
         if not os.path.isfile(filepath):
-            clean_up_and_exit("Not able to locate default file " + filename)
+            clean_up_and_exit("Not able to locate default file " + filename + " under " + filepath)
 
 
-def read_script_config():
+def read_script_config(default_tool_path):
     """This fucntion reads default CAD tools path from configuration file"""
     global config, cad_tools
     config = ConfigParser(interpolation=ExtendedInterpolation())
     config.read_dict(script_env_vars)
-    default_cad_tool_conf = os.path.join(
+    default_cad_tool_conf = default_tool_path
-        flow_script_dir, os.pardir, "misc", "fpgaflow_default_tool_path.conf"
-    )
     config.read_file(open(default_cad_tool_conf))
     if args.flow_config:
         config.read_file(open(args.flow_config))
@@ -905,18 +907,18 @@ def extract_vpr_stats(logfile, r_filename="vpr_stat", parse_section="vpr"):
     resultDict = {}
     for name, value in config.items(section):
         reg_string, filt_function = value.split(",")
-        match = re.search(reg_string[1:-1], vpr_log)
+        reg_result = re.findall(reg_string[1:-1], vpr_log)
-        if match:
+        if reg_result:
             try:
                 if "lambda" in filt_function.strip():
                     eval("ParseFunction = " + filt_function.strip())
-                    extract_val = ParseFunction(**match.groups())
+                    extract_val = ParseFunction(reg_result)
                 elif filt_function.strip() == "int":
-                    extract_val = int(match.group(1))
+                    extract_val = int(reg_result[-1])
                 elif filt_function.strip() == "float":
-                    extract_val = float(match.group(1))
+                    extract_val = float(reg_result[-1])
                 elif filt_function.strip() == "str":
-                    extract_val = str(match.group(1))
+                    extract_val = str(reg_result[-1])
                 elif filt_function.strip() == "scientific":
                     try:
                         mult = {
@@ -926,12 +928,12 @@ def extract_vpr_stats(logfile, r_filename="vpr_stat", parse_section="vpr"):
                             "K": 1e-3,
                             "M": 1e-6,
                             "G": 1e-9,
-                        }.get(match.group(2)[0], 1)
+                        }.get(reg_result[-1][1], 1)
                     except:
                         mult = 1
-                    extract_val = float(match.group(1)) * mult
+                    extract_val = float(reg_result[-1][0]) * mult
                 else:
-                    extract_val = match.group(1)
+                    extract_val = reg_result[-1]
             except:
                 logger.exception("Filter failed")
                 extract_val = "Filter Failed"

openfpga_flow/scripts/run_fpga_task.py

@@ -48,6 +48,7 @@ logger = logging.getLogger("OpenFPGA_Task_logs")
 # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
 # Read commandline arguments
 # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+task_script_dir = os.path.dirname(os.path.abspath(__file__))
 parser = argparse.ArgumentParser()
 parser.add_argument("tasks", nargs="+")
 parser.add_argument(
@@ -76,12 +77,17 @@ parser.add_argument("--continue_on_fail", action="store_true", help="Exit script
 parser.add_argument(
     "--show_thread_logs", action="store_true", help="Skips logs from running thread"
 )
+parser.add_argument(
+    "--default_tool_path",
+    type=str,
+    default=os.path.join(task_script_dir, os.pardir, "misc", "fpgaflow_default_tool_path.conf"),
+    help="The configuraton file contains paths to tools as well as keywords to be extracted from logs",
+)
 args = parser.parse_args()
 
 # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
 # Read script configuration file
 # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
-task_script_dir = os.path.dirname(os.path.abspath(__file__))
 script_env_vars = {
     "PATH": {
         "OPENFPGA_FLOW_PATH": task_script_dir,
@@ -407,6 +413,7 @@ def generate_each_task_actions(taskname):
                     task_conf=task_conf,
                 )
                 command += ["--flow_config", curr_task_conf_file]
+                command += ["--default_tool_path", args.default_tool_path]
                 flow_run_cmd_list.append(
                     {
                         "arch": arch,