riscv
/
OpenFPGA
mirror of https://github.com/lnis-uofu/OpenFPGA.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #993 from lnis-uofu/ext_exec 

New command ``ext_exec`` to allow system call; Automate bus group generation in openfpga shell scripts
This commit is contained in:
tangxifan 2023-01-11 20:14:47 -08:00 committed by GitHub
parent 9424806416 0375ec8501
commit dba59b0460
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 447 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/source/manual/openfpga_shell/openfpga_commands/basic_commands.rst
View File

@ -20,7 +20,7 @@ source
.. option:: --command_stream <string>
A string/file stream which contains the commands to be executed. Use quote(``"``) to split between commands. For example,
A string/file stream which contains the commands to be executed. Use quote(``"``) to group command and semicolumn(``;``) to split between commands. For example,
.. code-block::
@ -36,6 +36,19 @@ source
.. note:: If you are sourcing a file when running OpenFPGA in script mode, please turn on the batch mode here. See details in :ref:`launch_openfpga_shell`
ext_exec
~~~~~~~~
Run a system call for a command which is not in OpenFPGA shell
.. option:: --command <string>
A string stream which contains the command to be executed. Use quote(``"``) to group command. For example,
.. code-block::
ext_exec --command "ls -all"
exit
~~~~

71
openfpga/src/base/basic_command.cpp
View File

@ -7,58 +7,43 @@
#include "basic_command.h"
#include "command_exit_codes.h"
#include "openfpga_basic.h"
#include "openfpga_title.h"
/* begin namespace openfpga */
namespace openfpga {
static int source_existing_command(openfpga::Shell<OpenfpgaContext>* shell,
OpenfpgaContext& openfpga_ctx,
const Command& cmd,
const CommandContext& cmd_context) {
CommandOptionId opt_file = cmd.option("from_file");
CommandOptionId opt_batch_mode = cmd.option("batch_mode");
CommandOptionId opt_ss = cmd.option("command_stream");
/********************************************************************
* - Add a command to Shell environment: exec_external
* - Add associated options
* - Add command dependency
*******************************************************************/
static ShellCommandId add_openfpga_ext_exec_command(
openfpga::Shell<OpenfpgaContext>& shell,
const ShellCommandClassId& cmd_class_id,
const std::vector<ShellCommandId>& dependent_cmds) {
Command shell_cmd("ext_exec");
bool is_cmd_file = cmd_context.option_enable(cmd, opt_file);
std::string cmd_ss = cmd_context.option_value(cmd, opt_ss);
/* Add an option '--command_stream' */
CommandOptionId opt_cmdstream = shell_cmd.add_option(
"command", true,
"A string stream which contains the commands to be executed");
shell_cmd.set_option_require_value(opt_cmdstream, openfpga::OPT_STRING);
int status = CMD_EXEC_SUCCESS;
/* Add command to the Shell */
ShellCommandId shell_cmd_id = shell.add_command(
shell_cmd, "Source a string of commands or execute a script from a file");
shell.set_command_class(shell_cmd_id, cmd_class_id);
shell.set_command_execute_function(shell_cmd_id, call_external_command);
/* If a file is specified, run script mode of the shell, otherwise, */
if (is_cmd_file) {
shell->run_script_mode(cmd_ss.c_str(), openfpga_ctx,
cmd_context.option_enable(cmd, opt_batch_mode));
} else {
/* Split the string with ';' and run each command */
/* Remove the space at the end of the line
* So that we can check easily if there is a continued line in the end
*/
StringToken cmd_ss_tokenizer(cmd_ss);
/* Add command dependency to the Shell */
shell.set_command_dependency(shell_cmd_id, dependent_cmds);
for (std::string cmd_part : cmd_ss_tokenizer.split(";")) {
StringToken cmd_part_tokenizer(cmd_part);
cmd_part_tokenizer.rtrim(std::string(" "));
std::string single_cmd_line = cmd_part_tokenizer.data();
if (!single_cmd_line.empty()) {
status = shell->execute_command(single_cmd_line.c_str(), openfpga_ctx);
/* Check the execution status of the command,
* if fatal error happened, we should abort immediately
*/
if (CMD_EXEC_FATAL_ERROR == status) {
return CMD_EXEC_FATAL_ERROR;
}
}
}
}
return CMD_EXEC_SUCCESS;
return shell_cmd_id;
}
/********************************************************************
* - Add a command to Shell environment: repack
* - Add a command to Shell environment: source
* - Add associated options
* - Add command dependency
*******************************************************************/
@ -83,7 +68,7 @@ static ShellCommandId add_openfpga_source_command(
"batch_mode", false,
"Enable batch mode when executing the script from a file (not a string)");
/* Add command 'repack' to the Shell */
/* Add command to the Shell */
ShellCommandId shell_cmd_id = shell.add_command(
shell_cmd, "Source a string of commands or execute a script from a file");
shell.set_command_class(shell_cmd_id, cmd_class_id);
@ -126,6 +111,10 @@ void add_basic_commands(openfpga::Shell<OpenfpgaContext>& shell) {
add_openfpga_source_command(shell, basic_cmd_class,
std::vector<ShellCommandId>());
/* Add 'exec_external command which can run system call */
add_openfpga_ext_exec_command(shell, basic_cmd_class,
std::vector<ShellCommandId>());
/* Note:
* help MUST be the last to add because the linking to execute function will
* do a snapshot on the shell

75
openfpga/src/base/openfpga_basic.cpp Normal file
View File

@ -0,0 +1,75 @@
/********************************************************************
* Add basic commands to the OpenFPGA shell interface, including:
* - exit
* - version
* - help
*******************************************************************/
#include "openfpga_basic.h"
#include "command_exit_codes.h"
#include "openfpga_title.h"
/* begin namespace openfpga */
namespace openfpga {
int source_existing_command(openfpga::Shell<OpenfpgaContext>* shell,
OpenfpgaContext& openfpga_ctx, const Command& cmd,
const CommandContext& cmd_context) {
CommandOptionId opt_file = cmd.option("from_file");
CommandOptionId opt_batch_mode = cmd.option("batch_mode");
CommandOptionId opt_ss = cmd.option("command_stream");
bool is_cmd_file = cmd_context.option_enable(cmd, opt_file);
std::string cmd_ss = cmd_context.option_value(cmd, opt_ss);
int status = CMD_EXEC_SUCCESS;
/* If a file is specified, run script mode of the shell, otherwise, */
if (is_cmd_file) {
shell->run_script_mode(cmd_ss.c_str(), openfpga_ctx,
cmd_context.option_enable(cmd, opt_batch_mode));
} else {
/* Split the string with ';' and run each command */
/* Remove the space at the end of the line
* So that we can check easily if there is a continued line in the end
*/
StringToken cmd_ss_tokenizer(cmd_ss);
for (std::string cmd_part : cmd_ss_tokenizer.split(";")) {
StringToken cmd_part_tokenizer(cmd_part);
cmd_part_tokenizer.rtrim(std::string(" "));
std::string single_cmd_line = cmd_part_tokenizer.data();
if (!single_cmd_line.empty()) {
status = shell->execute_command(single_cmd_line.c_str(), openfpga_ctx);
/* Check the execution status of the command,
* if fatal error happened, we should abort immediately
*/
if (CMD_EXEC_FATAL_ERROR == status) {
return CMD_EXEC_FATAL_ERROR;
}
}
}
}
return CMD_EXEC_SUCCESS;
}
/** Call an external command using system call */
int call_external_command(const Command& cmd,
const CommandContext& cmd_context) {
CommandOptionId opt_ss = cmd.option("command");
std::string cmd_ss = cmd_context.option_value(cmd, opt_ss);
/* First test if command processor is available and then execute */
if (!system(NULL)) {
VTR_LOG("Processer is not available");
return CMD_EXEC_FATAL_ERROR;
}
return system(cmd_ss.c_str());
}
} /* end namespace openfpga */

28
openfpga/src/base/openfpga_basic.h Normal file
View File

@ -0,0 +1,28 @@
#ifndef OPENFPGA_BASIC_H
#define OPENFPGA_BASIC_H
/********************************************************************
* Include header files that are required by function declaration
*******************************************************************/
#include "command.h"
#include "command_context.h"
#include "openfpga_context.h"
#include "shell.h"
/********************************************************************
* Function declaration
*******************************************************************/
/* begin namespace openfpga */
namespace openfpga {
int source_existing_command(openfpga::Shell<OpenfpgaContext>* shell,
OpenfpgaContext& openfpga_ctx, const Command& cmd,
const CommandContext& cmd_context);
int call_external_command(const Command& cmd,
const CommandContext& cmd_context);
} /* end namespace openfpga */
#endif

65
openfpga_flow/openfpga_shell_scripts/auto_bus_group_example_script.openfpga Normal file
View File

@ -0,0 +1,65 @@
# Run VPR for the 'and' design
#--write_rr_graph example_rr_graph.xml
vpr ${VPR_ARCH_FILE} ${VPR_TESTBENCH_BLIF} --clock_modeling ideal
# Read OpenFPGA architecture definition
read_openfpga_arch -f ${OPENFPGA_ARCH_FILE}
# Read OpenFPGA simulation settings
read_openfpga_simulation_setting -f ${OPENFPGA_SIM_SETTING_FILE}
# Annotate the OpenFPGA architecture to VPR data base
# to debug use --verbose options
link_openfpga_arch --sort_gsb_chan_node_in_edges
# Check and correct any naming conflicts in the BLIF netlist
check_netlist_naming_conflict --fix --report ./netlist_renaming.xml
# Apply fix-up to Look-Up Table truth tables based on packing results
lut_truth_table_fixup
# Build the module graph
# - Enabled compression on routing architecture modules
# - Enable pin duplication on grid modules
build_fabric --compress_routing #--verbose
# Write the fabric hierarchy of module graph to a file
# This is used by hierarchical PnR flows
write_fabric_hierarchy --file ./fabric_hierarchy.txt
# Repack the netlist to physical pbs
# This must be done before bitstream generator and testbench generation
# Strongly recommend it is done after all the fix-up have been applied
repack #--verbose
# Build the bitstream
# - Output the fabric-independent bitstream to a file
build_architecture_bitstream --verbose --write_file fabric_independent_bitstream.xml
# Build fabric-dependent bitstream
build_fabric_bitstream --verbose
# Write fabric-dependent bitstream
write_fabric_bitstream --file fabric_bitstream.bit --format plain_text
# Write the Verilog netlist for FPGA fabric
# - Enable the use of explicit port mapping in Verilog netlist
write_fabric_verilog --file ./SRC --explicit_port_mapping --include_timing --verbose
# Generate a bus group file by calling an external python script
ext_exec --command "python3 ../../../../config/bus_group_gen.py --task ../../../../config/counter8_bus_group_task.yaml"
# Write the Verilog testbench for FPGA fabric
# - We suggest the use of same output directory as fabric Verilog netlists
# - Must specify the reference benchmark file if you want to output any testbenches
# - Enable top-level testbench which is a full verification including programming circuit and core logic of FPGA
# - Enable pre-configured top-level testbench which is a fast verification skipping programming phase
# - Simulation ini file is optional and is needed only when you need to interface different HDL simulators using openfpga flow-run scripts
write_preconfigured_fabric_wrapper --embed_bitstream iverilog --file ./SRC ${OPENFPGA_VERILOG_PORT_MAPPING} --pin_constraints_file ${OPENFPGA_PIN_CONSTRAINTS_FILE} --bus_group_file ${OPENFPGA_BUS_GROUP_FILE}
write_preconfigured_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} ${OPENFPGA_VERILOG_PORT_MAPPING} --pin_constraints_file ${OPENFPGA_PIN_CONSTRAINTS_FILE} --bus_group_file ${OPENFPGA_BUS_GROUP_FILE}
# Finish and exit OpenFPGA
exit
# Note :
# To run verification at the end of the flow maintain source in ./SRC directory

1
openfpga_flow/regression_test_scripts/basic_reg_test.sh
View File

@ -190,6 +190,7 @@ run-task basic_tests/bus_group/preconfig_testbench_explicit_mapping $@
run-task basic_tests/bus_group/preconfig_testbench_implicit_mapping $@
run-task basic_tests/bus_group/full_testbench_explicit_mapping $@
run-task basic_tests/bus_group/full_testbench_implicit_mapping $@
run-task basic_tests/bus_group/auto_gen_bus_group $@
echo -e "Testing fix pins features";
run-task basic_tests/io_constraints/fix_pins $@

173
openfpga_flow/tasks/basic_tests/bus_group/auto_gen_bus_group/config/bus_group_gen.py Normal file
View File

@ -0,0 +1,173 @@
#####################################################################
# A script to create a bus group file based on an input verilog file
# The bug group file is an input required by OpenFPGA
#####################################################################
import os
from os.path import dirname, abspath
import argparse
import logging
import subprocess
import hashlib
import yaml
import pyverilog
from pyverilog.dataflow.dataflow_analyzer import VerilogDataflowAnalyzer
from xml.dom import minidom
#####################################################################
# Error codes
#####################################################################
error_codes = {
"SUCCESS": 0,
"MD5_ERROR": 1,
"OPTION_ERROR": 2,
"FILE_ERROR": 3
}
#####################################################################
# Initialize logger
#####################################################################
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO);
#####################################################################
# Generate the string for a Verilog port
#####################################################################
def gen_verilog_port_str(port_name, msb, lsb):
port_str = str(port_name) + "[" + str(msb) + ":" + str(lsb) + "]"
return port_str
#####################################################################
# Generate the string for a flatten Verilog port
#####################################################################
def gen_flatten_verilog_port_str(port_name, pin_id):
port_str = str(port_name) + "_" + str(pin_id) + "_"
return port_str
#####################################################################
# Parse a verilog file and collect bus port information
#####################################################################
def parse_verilog_file_bus_ports(verilog_files, top_module):
# Check if verilog file exists
verilog_file_list = []
for verilog_f in verilog_files:
print(verilog_f)
verilog_f_abspath = os.path.abspath(verilog_f["name"])
if not os.path.exists(verilog_f_abspath):
raise IOError("file not found: " + verilog_f_abspath)
verilog_file_list.append(verilog_f_abspath)
# Parse verilog file
analyzer = VerilogDataflowAnalyzer(verilog_file_list, top_module)
analyzer.generate()
# Get port information
terms = analyzer.getTerms()
# Create XML tree
xml = minidom.Document()
bus_group = xml.createElement("bus_group")
xml.appendChild(bus_group)
for tk, tv in sorted(terms.items(), key=lambda x: str(x[0])):
logging.debug(tv.name)
logging.debug(tv.termtype)
logging.debug("[" + str(tv.lsb) + ":" + str(tv.msb) + "]")
for tk, tv in sorted(terms.items(), key=lambda x: str(x[0])):
# Skip ports that do not belong to top module
if (top_module != str(tv.name).split(".")[-2]):
continue
port_name = str(tv.name).split(".")[-1]
# Skip minus lsb or msb, which are in don't care set
if (("Minus" == str(tv.lsb)) or ("Minus" == str(tv.msb))):
continue
port_lsb = int(str(tv.lsb))
port_msb = int(str(tv.msb))
# Only care input and outports
if ((not ("Input" in tv.termtype)) and (not ("Output" in tv.termtype))):
continue
# Only care bus (msb - lsb > 0)
if (abs(port_lsb - port_msb) == 0):
continue
# Reaching here, this is a bus port we need
# Get the second part of the name, which is the port name
cur_bus = xml.createElement("bus")
cur_bus.setAttribute("name", gen_verilog_port_str(port_name, port_msb, port_lsb))
# Get if this is little endian or not
cur_bus.setAttribute("big_endian", "false")
if (port_lsb > port_msb):
cur_bus.setAttribute("big_endian", "true")
bus_group.appendChild(cur_bus)
# Add all the pins
for ipin in range(min([port_msb, port_lsb]), max([port_msb, port_lsb]) + 1):
cur_pin = xml.createElement("pin")
cur_pin.setAttribute('id', str(ipin))
cur_pin.setAttribute('name', gen_flatten_verilog_port_str(port_name, ipin))
cur_bus.appendChild(cur_pin)
return xml, bus_group
#####################################################################
# Generate bus group files with a given task list
#####################################################################
def generate_bus_group_files(task_db):
# Iterate over all the tasks
for verilog_fname in task_db.keys():
space_limit = 120
log_str = "Parsing verilog file: "
top_module_name = task_db[verilog_fname]["top_module"]
logging_space = "." * (space_limit - len(log_str) - len(top_module_name))
logging.info(log_str + top_module_name)
xml, bus_group_data = parse_verilog_file_bus_ports(task_db[verilog_fname]["source"], top_module_name)
logging.info(log_str + top_module_name + logging_space + "Done")
# Write bus ports to an XML file
bus_group_frelname = task_db[verilog_fname]["bus_group_file"]
bus_group_fname = os.path.abspath(bus_group_frelname)
log_str = "Writing bus group file:"
logging_space = "." * (space_limit - len(log_str) - len(bus_group_frelname))
logging.info(log_str + bus_group_frelname)
xml_str = xml.toprettyxml(indent="\t")
with open(bus_group_fname, "w") as bus_group_f:
bus_group_f.write(xml_str)
logging.info(log_str + bus_group_frelname + logging_space + "Done")
#####################################################################
# Read task list from a yaml file
#####################################################################
def read_yaml_to_task_database(yaml_filename):
task_db = {}
with open(yaml_filename, 'r') as stream:
try:
task_db = yaml.load(stream, Loader=yaml.FullLoader)
logging.info("Found " + str(len(task_db)) + " tasks to create symbolic links")
except yaml.YAMLError as exc:
logging.error(exc)
exit(error_codes["FILE_ERROR"]);
return task_db
#####################################################################
# Write result database to a yaml file
#####################################################################
def write_result_database_to_yaml(result_db, yaml_filename):
with open(yaml_filename, 'w') as yaml_file:
yaml.dump(result_db, yaml_file, default_flow_style=False)
#####################################################################
# Main function
#####################################################################
if __name__ == '__main__':
# Execute when the module is not initialized from an import statement
# Parse the options and apply sanity checks
parser = argparse.ArgumentParser(description='Create bus group files for Verilog inputs')
parser.add_argument('--task_list',
required=True,
help='Configuration file in YAML format which contains a list of input Verilog and output bus group files')
args = parser.parse_args()
# Create a database for tasks
task_db = {}
task_db = read_yaml_to_task_database(args.task_list)
# Generate links based on the task list in database
generate_bus_group_files(task_db)
logging.info("Created " + str(len(task_db)) + " bus group files")
exit(error_codes["SUCCESS"])

5
openfpga_flow/tasks/basic_tests/bus_group/auto_gen_bus_group/config/counter8_bus_group_task.yaml Normal file
View File

@ -0,0 +1,5 @@
counter8:
source:
- name: counter_output_verilog.v
top_module: counter
bus_group_file: bus_group.xml

7
openfpga_flow/tasks/basic_tests/bus_group/auto_gen_bus_group/config/pin_constraints_reset.xml Normal file
View File

@ -0,0 +1,7 @@
<pin_constraints>
<!-- For a given .blif file, we want to assign
- the reset signal to the op_reset[0] port of the FPGA fabric
-->
<set_io pin="op_reset[0]" net="reset"/>
</pin_constraints>

48
openfpga_flow/tasks/basic_tests/bus_group/auto_gen_bus_group/config/task.conf Normal file
View File

@ -0,0 +1,48 @@
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
# Configuration file for running experiments
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
# timeout_each_job : FPGA Task script splits fpga flow into multiple jobs
# Each job execute fpga_flow script on combination of architecture & benchmark
# timeout_each_job is timeout for each job
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
[GENERAL]
run_engine=openfpga_shell
power_tech_file = ${PATH:OPENFPGA_PATH}/openfpga_flow/tech/PTM_45nm/45nm.xml
power_analysis = false
spice_output=false
verilog_output=true
timeout_each_job = 20*60
fpga_flow=yosys_vpr
[OpenFPGA_SHELL]
openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_shell_scripts/auto_bus_group_example_script.openfpga
openfpga_arch_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_arch/k6_frac_N10_adder_chain_dpram8K_dsp36_fracff_40nm_openfpga.xml
openfpga_sim_setting_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_simulation_settings/fixed_sim_openfpga.xml
openfpga_verilog_port_mapping=--explicit_port_mapping
[ARCHITECTURES]
arch0=${PATH:OPENFPGA_PATH}/openfpga_flow/vpr_arch/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_fracff_40nm.xml
[BENCHMARKS]
bench0=${PATH:OPENFPGA_PATH}/openfpga_flow/benchmarks/micro_benchmark/counters/counter_8bit_async_reset/counter.v
[SYNTHESIS_PARAM]
# Yosys script parameters
bench_yosys_cell_sim_verilog_common=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_yosys_techlib/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_fracff_40nm_cell_sim.v
bench_yosys_dff_map_verilog_common=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_yosys_techlib/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_fracff_40nm_dff_map.v
bench_yosys_bram_map_rules_common=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_yosys_techlib/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_40nm_bram.txt
bench_yosys_bram_map_verilog_common=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_yosys_techlib/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_40nm_bram_map.v
bench_yosys_dsp_map_verilog_common=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_yosys_techlib/k6_frac_N10_tileable_adder_chain_dpram8K_dsp36_40nm_dsp_map.v
bench_yosys_dsp_map_parameters_common=-D DSP_A_MAXWIDTH=36 -D DSP_B_MAXWIDTH=36 -D DSP_A_MINWIDTH=2 -D DSP_B_MINWIDTH=2 -D DSP_NAME=mult_36x36
bench_read_verilog_options_common = -nolatches
bench_yosys_common=${PATH:OPENFPGA_PATH}/openfpga_flow/misc/ys_tmpl_yosys_vpr_dff_flow.ys
bench_yosys_rewrite_common=${PATH:OPENFPGA_PATH}/openfpga_flow/misc/ys_tmpl_yosys_vpr_flow_with_rewrite.ys
bench0_top = counter
bench0_openfpga_pin_constraints_file=${PATH:TASK_DIR}/config/pin_constraints_reset.xml
bench0_openfpga_bus_group_file=bus_group.xml
[SCRIPT_PARAM_MIN_ROUTE_CHAN_WIDTH]
end_flow_with_test=
vpr_fpga_verilog_formal_verification_top_netlist=

1
requirements.txt
View File

@ -1,6 +1,7 @@
envyaml==1.0.201125
humanize==3.1.0
coloredlogs==9.1
pyverilog
# Python linter and formatter
click==8.0.2 # Our version of black needs an older version of click (https://stackoverflow.com/questions/71673404/importerror-cannot-import-name-unicodefun-from-click)