From 1554f583b7ddf307c1d118c4bb0fa1f44a997acf Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 17:22:41 -0700 Subject: [PATCH 1/9] [Flow] Now support explicit variable file for task --- openfpga_flow/scripts/run_fpga_flow.py | 13 +++++++++++++ openfpga_flow/scripts/run_fpga_task.py | 3 +++ 2 files changed, 16 insertions(+) diff --git a/openfpga_flow/scripts/run_fpga_flow.py b/openfpga_flow/scripts/run_fpga_flow.py index a331d506d..a84ee915f 100644 --- a/openfpga_flow/scripts/run_fpga_flow.py +++ b/openfpga_flow/scripts/run_fpga_flow.py @@ -14,9 +14,11 @@ import time from datetime import timedelta import shlex import glob +import json import argparse from configparser import ConfigParser, ExtendedInterpolation import logging +from envyaml import EnvYAML import glob import subprocess import threading @@ -85,6 +87,8 @@ parser.add_argument('--openfpga_shell_template', type=str, help="Sample openfpga shell script") parser.add_argument('--openfpga_arch_file', type=str, help="Openfpga architecture file for shell") +parser.add_argument('--arch_variable_file', type=str, default=None, + help="Openfpga architecture file for shell") # parser.add_argument('--openfpga_sim_setting_file', type=str, # help="Openfpga simulation file for shell") # parser.add_argument('--external_fabric_key_file', type=str, @@ -313,6 +317,15 @@ def read_script_config(): clean_up_and_exit("Missing CAD_TOOLS_PATH in openfpga_flow config") cad_tools = config["CAD_TOOLS_PATH"] + if args.arch_variable_file: + _, file_extension = os.path.splitext(args.arch_variable_file) + if file_extension in [".yml", ".yaml"]: + script_env_vars["PATH"].update( + EnvYAML(args.arch_variable_file, include_environment=False)) + if file_extension in [".json", ]: + with open(args.arch_variable_file, "r") as fp: + script_env_vars["PATH"].update(json.load(fp)) + def validate_command_line_arguments(): """ diff --git a/openfpga_flow/scripts/run_fpga_task.py b/openfpga_flow/scripts/run_fpga_task.py index 6ab46e1df..7fb0346dc 100644 --- a/openfpga_flow/scripts/run_fpga_task.py +++ b/openfpga_flow/scripts/run_fpga_task.py @@ -371,6 +371,9 @@ def create_run_command(curr_job_dir, archfile, benchmark_obj, param, task_conf): command += ["--power"] command += ["--power_tech", task_gc.get("power_tech_file")] + if task_gc.get("arch_variable_file"): + command += ["--arch_variable_file", task_gc.get("arch_variable_file")] + if task_gc.getboolean("spice_output"): command += ["--vpr_fpga_spice"] From 1d993296d8afc4bec0c85bd85220d05a0bb63e41 Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 17:25:12 -0700 Subject: [PATCH 2/9] [Flow] Example of using test variable in task conf --- .../k6_frac_N8_40nm_openfpga.xml | 47 ++++++++++--------- .../config/design_variables.yml | 1 + .../generate_fabric/config/task.conf | 1 + 3 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml diff --git a/openfpga_flow/openfpga_arch/k6_frac_N8_40nm_openfpga.xml b/openfpga_flow/openfpga_arch/k6_frac_N8_40nm_openfpga.xml index 74d1d9933..59b00016f 100644 --- a/openfpga_flow/openfpga_arch/k6_frac_N8_40nm_openfpga.xml +++ b/openfpga_flow/openfpga_arch/k6_frac_N8_40nm_openfpga.xml @@ -1,10 +1,11 @@ @@ -101,7 +102,8 @@ - + + @@ -109,7 +111,8 @@ - + + @@ -140,14 +143,14 @@ - - - - - - - - + + + + + + + + @@ -165,14 +168,14 @@ - - - - - - - - + + + + + + + + @@ -199,9 +202,9 @@ - - - + + + diff --git a/openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml b/openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml new file mode 100644 index 000000000..de1dbf341 --- /dev/null +++ b/openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml @@ -0,0 +1 @@ +TEST_VARIABLE: 100 \ No newline at end of file diff --git a/openfpga_flow/tasks/basic_tests/generate_fabric/config/task.conf b/openfpga_flow/tasks/basic_tests/generate_fabric/config/task.conf index 9c3fd2442..68cee7380 100644 --- a/openfpga_flow/tasks/basic_tests/generate_fabric/config/task.conf +++ b/openfpga_flow/tasks/basic_tests/generate_fabric/config/task.conf @@ -14,6 +14,7 @@ spice_output=false verilog_output=true timeout_each_job = 20*60 fpga_flow=vpr_blif +arch_variable_file=${PATH:TASK_DIR}/design_variables.yml [OpenFPGA_SHELL] openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/OpenFPGAShellScripts/generate_fabric_example_script.openfpga From 2e4769feb972ae2f5dfad118392aebb58f6df7a9 Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 18:10:26 -0700 Subject: [PATCH 3/9] [Build] Added python depenencies in CI/CD build --- .github/workflows/install_dependency.sh | 2 ++ requirements.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/.github/workflows/install_dependency.sh b/.github/workflows/install_dependency.sh index 0e663ef26..c1335b5f8 100644 --- a/.github/workflows/install_dependency.sh +++ b/.github/workflows/install_dependency.sh @@ -46,3 +46,5 @@ sudo apt-get install g++-9 sudo apt-get install gcc-9 sudo apt-get install clang-6.0 sudo apt-get install clang-8 +# Python dependencies +python3 -m pip install requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..06e50e114 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +envyaml==1.0.201125 \ No newline at end of file From bc90d5363111f1799dffc4a4c35eef940d2c229f Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 18:35:15 -0700 Subject: [PATCH 4/9] [Build] Corrected requirements file path --- .github/workflows/install_dependency.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install_dependency.sh b/.github/workflows/install_dependency.sh index c1335b5f8..1dbe17c7c 100644 --- a/.github/workflows/install_dependency.sh +++ b/.github/workflows/install_dependency.sh @@ -47,4 +47,4 @@ sudo apt-get install gcc-9 sudo apt-get install clang-6.0 sudo apt-get install clang-8 # Python dependencies -python3 -m pip install requirements.txt +python3 -m pip install /home/runner/work/OpenFPGA/OpenFPGA/requirements.txt From febdd0896c8d5522084fcdea637fba5ca1df0ab0 Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 18:44:26 -0700 Subject: [PATCH 5/9] [Build] Bux fix --- .github/workflows/install_dependency.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install_dependency.sh b/.github/workflows/install_dependency.sh index 1dbe17c7c..eba89f2a7 100644 --- a/.github/workflows/install_dependency.sh +++ b/.github/workflows/install_dependency.sh @@ -47,4 +47,4 @@ sudo apt-get install gcc-9 sudo apt-get install clang-6.0 sudo apt-get install clang-8 # Python dependencies -python3 -m pip install /home/runner/work/OpenFPGA/OpenFPGA/requirements.txt +python3 -m pip install -r /home/runner/work/OpenFPGA/OpenFPGA/requirements.txt From 91d3f289a8f12b7eead857ed5f91a12c9bbec3d5 Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 21:11:15 -0700 Subject: [PATCH 6/9] [Build] Added python setup tools --- .github/workflows/install_dependency.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/install_dependency.sh b/.github/workflows/install_dependency.sh index eba89f2a7..c4f70c29f 100644 --- a/.github/workflows/install_dependency.sh +++ b/.github/workflows/install_dependency.sh @@ -26,6 +26,7 @@ sudo apt-get install libxft-dev sudo apt-get install libxml++2.6-dev sudo apt-get install perl sudo apt-get install python +sudo apt-get install python3-setuptools sudo apt-get install python-lxml sudo apt-get install texinfo sudo apt-get install time From 59bd7d0f186157b15afb3a2ad612ad4225153b81 Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 22:09:36 -0700 Subject: [PATCH 7/9] [Flow] Changed substitute to safe_sustitute option --- openfpga_flow/scripts/run_fpga_flow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openfpga_flow/scripts/run_fpga_flow.py b/openfpga_flow/scripts/run_fpga_flow.py index c7609aa21..21959398a 100644 --- a/openfpga_flow/scripts/run_fpga_flow.py +++ b/openfpga_flow/scripts/run_fpga_flow.py @@ -424,7 +424,7 @@ def prepare_run_directory(run_dir): arch_filename = os.path.basename(args.arch_file) args.arch_file = os.path.join(run_dir, "arch", arch_filename) with open(args.arch_file, 'w', encoding='utf-8') as archfile: - archfile.write(tmpl.substitute(script_env_vars["PATH"])) + archfile.write(tmpl.safe_substitute(script_env_vars["PATH"])) if (args.openfpga_arch_file): tmpl = Template( @@ -432,7 +432,7 @@ def prepare_run_directory(run_dir): arch_filename = os.path.basename(args.openfpga_arch_file) args.openfpga_arch_file = os.path.join(run_dir, "arch", arch_filename) with open(args.openfpga_arch_file, 'w', encoding='utf-8') as archfile: - archfile.write(tmpl.substitute(script_env_vars["PATH"])) + archfile.write(tmpl.safe_substitute(script_env_vars["PATH"])) # Sanitize provided openshell template, if provided if (args.openfpga_shell_template): @@ -491,7 +491,7 @@ def run_yosys_with_abc(): cad_tools["misc_dir"], "ys_tmpl_yosys_vpr_flow.ys") tmpl = Template(open(yosys_template, encoding='utf-8').read()) with open("yosys.ys", 'w') as archfile: - archfile.write(tmpl.substitute(ys_params)) + archfile.write(tmpl.safe_substitute(ys_params)) try: with open('yosys_output.txt', 'w+') as output: process = subprocess.run([cad_tools["yosys_path"], 'yosys.ys'], @@ -714,7 +714,7 @@ def run_openfpga_shell(): path_variables[tmpVar] = OpenFPGAArgs[indx+1] with open(args.top_module+"_run.openfpga", 'w', encoding='utf-8') as archfile: - archfile.write(tmpl.substitute(path_variables)) + archfile.write(tmpl.safe_substitute(path_variables)) command = [cad_tools["openfpga_shell_path"], "-f", args.top_module+"_run.openfpga"] run_command("OpenFPGA Shell Run", "openfpgashell.log", command) @@ -734,7 +734,7 @@ def run_standard_vpr(bench_blif, fixed_chan_width, logfile, route_only=False): ] if not args.disp: command += ["--disp", "off"] - else: + else: command += ["--disp", "on"] if route_only: From 7db030018cb73f21b9439100cf97994b5ca8d2ce Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Wed, 25 Nov 2020 22:44:40 -0700 Subject: [PATCH 8/9] [Bug] Fixed variable file location --- .../generate_fabric/{config => }/design_variables.yml | 0 requirements.txt | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename openfpga_flow/tasks/basic_tests/generate_fabric/{config => }/design_variables.yml (100%) diff --git a/openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml b/openfpga_flow/tasks/basic_tests/generate_fabric/design_variables.yml similarity index 100% rename from openfpga_flow/tasks/basic_tests/generate_fabric/config/design_variables.yml rename to openfpga_flow/tasks/basic_tests/generate_fabric/design_variables.yml diff --git a/requirements.txt b/requirements.txt index 06e50e114..7c0776b6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -envyaml==1.0.201125 \ No newline at end of file +envyaml==1.0.201125 +humanize==3.1.0 \ No newline at end of file From 289d9d21695b0fd25c1155e4ec8ef02d79fa6b4b Mon Sep 17 00:00:00 2001 From: ganeshgore Date: Thu, 3 Dec 2020 12:24:24 -0700 Subject: [PATCH 9/9] [Bugfix] Honors yosys_tmpl parameter in flow script --- openfpga_flow/scripts/run_fpga_flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfpga_flow/scripts/run_fpga_flow.py b/openfpga_flow/scripts/run_fpga_flow.py index 21959398a..3501c28c0 100644 --- a/openfpga_flow/scripts/run_fpga_flow.py +++ b/openfpga_flow/scripts/run_fpga_flow.py @@ -93,7 +93,7 @@ parser.add_argument('--arch_variable_file', type=str, default=None, # help="Openfpga simulation file for shell") # parser.add_argument('--external_fabric_key_file', type=str, # help="Key file for shell") -parser.add_argument('--yosys_tmpl', type=str, +parser.add_argument('--yosys_tmpl', type=str, default=None, help="Alternate yosys template, generates top_module.blif") parser.add_argument('--disp', action="store_true", help="Open display while running VPR") @@ -487,7 +487,7 @@ def run_yosys_with_abc(): "LUT_SIZE": lut_size, "OUTPUT_BLIF": args.top_module+"_yosys_out.blif", } - yosys_template = os.path.join( + yosys_template = args.yosys_tmpl if args.yosys_tmpl else os.path.join( cad_tools["misc_dir"], "ys_tmpl_yosys_vpr_flow.ys") tmpl = Template(open(yosys_template, encoding='utf-8').read()) with open("yosys.ys", 'w') as archfile: