Merge pull request #970 from lnis-uofu/shell_source

Now support calling other commands through a string or an external file inside a script/shell

docs/source/manual/openfpga_shell/openfpga_commands/basic_commands.rst

@@ -13,6 +13,29 @@ help
 
   Show help desk to list all the available commands
 
+source
+~~~~~~
+
+  Run a set of existing commands from a string stream or a file
+
+  .. option:: --command_stream <string>
+
+    A string/file stream which contains the commands to be executed. Use quote(``"``) to split between commands. For example,
+
+  .. code-block::
+
+     source --command_stream "help;exit;"
+
+  .. option:: --from_file
+
+    Specify the command stream comes from a file. When selected, the file will be parsed as a regular script following the OpenFPGA script file format. See details in :ref:`openfpga_script_format`
+
+  .. option:: --batch_mode
+
+    Enable batch mode when executing the script from a file. Valid only when ``--from_file`` is enabled.
+
+    .. note:: If you are sourcing a file when running OpenFPGA in script mode, please turn on the batch mode here. See details in :ref:`launch_openfpga_shell`
+
 exit
 ~~~~
 

libs/libopenfpgashell/src/shell.h

@@ -71,6 +71,7 @@ class Shell {
     SHORT,
     BUILTIN,
     MACRO,
+    WRAPPER,
     NUM_EXEC_FUNC_TYPES
   };
 
@@ -154,6 +155,12 @@ class Shell {
   void set_command_execute_function(const ShellCommandId& cmd_id,
                                     std::function<int(int, char**)> exec_func);
 
+  /* Wrapper function, which calls other command thru shell's APIs */
+  void set_command_execute_function(
+    const ShellCommandId& cmd_id,
+    std::function<int(Shell<T>*, T&, const Command&, const CommandContext&)>
+      exec_func);
+
   void set_command_dependency(
     const ShellCommandId& cmd_id,
     const std::vector<ShellCommandId>& cmd_dependency);
@@ -235,6 +242,9 @@ class Shell {
     command_builtin_execute_functions_;
   vtr::vector<ShellCommandId, std::function<int(int, char**)>>
     command_macro_execute_functions_;
+  vtr::vector<ShellCommandId, std::function<int(Shell<T>*, T&, const Command&,
+                                                const CommandContext&)>>
+    command_wrapper_execute_functions_;
 
   /* Type of execute functions for each command.
    * This is supposed to be an internal data ONLY

libs/libopenfpgashell/src/shell.tpp

@@ -136,6 +136,7 @@ ShellCommandId Shell<T>::add_command(const Command& cmd, const char* descr) {
   command_short_const_execute_functions_.emplace_back();
   command_short_execute_functions_.emplace_back();
   command_builtin_execute_functions_.emplace_back();
+  command_wrapper_execute_functions_.emplace_back();
   command_macro_execute_functions_.emplace_back();
   command_status_.push_back(CMD_EXEC_NONE); /* By default, the command should be marked as fatal error as it has been never executed */
   command_dependencies_.emplace_back();
@@ -207,6 +208,14 @@ void Shell<T>::set_command_execute_function(const ShellCommandId& cmd_id,
   command_macro_execute_functions_[cmd_id] = exec_func;
 }
 
+template<class T>
+void Shell<T>::set_command_execute_function(const ShellCommandId& cmd_id, 
+                                            std::function<int(Shell<T>*, T&, const Command&, const CommandContext&)> exec_func) {
+  VTR_ASSERT(true == valid_command_id(cmd_id));
+  command_execute_function_types_[cmd_id] = WRAPPER;
+  command_wrapper_execute_functions_[cmd_id] = exec_func;
+}
+
 template<class T>
 void Shell<T>::set_command_dependency(const ShellCommandId& cmd_id,
                                       const std::vector<ShellCommandId>& dependent_cmds) {
@@ -477,9 +486,45 @@ void Shell<T>::exit(const int& init_err) const {
 template <class T>
 int Shell<T>::execute_command(const char* cmd_line,
                                T& common_context) {
-  /* Tokenize the line */
   openfpga::StringToken tokenizer(cmd_line);  
-  std::vector<std::string> tokens = tokenizer.split(" ");
+  /* Do not split the string in each quote "", as they should be a piece */
+  std::vector<size_t> quote_anchors;
+  size_t quote_found = tokenizer.data().find("\"");
+  while (std::string::npos != quote_found) {
+    quote_anchors.push_back(quote_found);
+    quote_found = tokenizer.data().find("\"", quote_found+1);
+  }
+  /* Quote should be not be started with! */
+  if (!quote_anchors.empty() && quote_anchors.front() == 0) {
+    VTR_LOG("Quotes (\") should NOT be the first charactor in command line: '%s'\n", cmd_line);
+    return CMD_EXEC_FATAL_ERROR;
+  }
+  /* Quotes must be in pairs! */
+  if (0 != quote_anchors.size() % 2) {
+    VTR_LOG("Quotes (\") are not in pair in command line: '%s'\n", cmd_line);
+    return CMD_EXEC_FATAL_ERROR;
+  }
+  /* Tokenize the line based on anchors */
+  std::vector<std::string> tokens;
+  if (quote_anchors.empty()) {
+    tokens = tokenizer.split(" ");
+  } else {
+    /* There are pairs of quotes, identify the chunk which should be split*/
+    std::vector<std::string> token_chunks = tokenizer.split("\"");
+    for (size_t ichunk = 0; ichunk < token_chunks.size(); ichunk++) {
+      /* Chunk with even index (including the first) is always out of two quote -> Split!
+       * Chunk with odd index is always between two quotes -> Do not split!
+       */
+      if (ichunk % 2 == 0) {
+        openfpga::StringToken chunk_tokenizer(token_chunks[ichunk]);  
+        for (std::string curr_token : chunk_tokenizer.split(" ")) {
+          tokens.push_back(curr_token);
+        }
+      } else {
+        tokens.push_back(token_chunks[ichunk]);
+      }
+    }
+  } 
 
   /* Find if the command name is valid */
   ShellCommandId cmd_id = command(tokens[0]);
@@ -541,6 +586,9 @@ int Shell<T>::execute_command(const char* cmd_line,
 
   /* Execute the command depending on the type of function ! */ 
   switch (command_execute_function_types_[cmd_id]) {
+  case WRAPPER:
+    command_status_[cmd_id] = command_wrapper_execute_functions_[cmd_id](this, common_context, commands_[cmd_id], command_contexts_[cmd_id]);
+    break;
   case CONST_STANDARD:
     command_status_[cmd_id] = command_const_execute_functions_[cmd_id](common_context, commands_[cmd_id], command_contexts_[cmd_id]);
     break;

openfpga/src/base/basic_command.cpp

@@ -6,11 +6,95 @@
  *******************************************************************/
 #include "basic_command.h"
 
+#include "command_exit_codes.h"
 #include "openfpga_title.h"
 
 /* begin namespace openfpga */
 namespace openfpga {
 
+static int source_existing_command(openfpga::Shell<OpenfpgaContext>* shell,
+                                   OpenfpgaContext& openfpga_ctx,
+                                   const Command& cmd,
+                                   const CommandContext& cmd_context) {
+  CommandOptionId opt_file = cmd.option("from_file");
+  CommandOptionId opt_batch_mode = cmd.option("batch_mode");
+  CommandOptionId opt_ss = cmd.option("command_stream");
+
+  bool is_cmd_file = cmd_context.option_enable(cmd, opt_file);
+  std::string cmd_ss = cmd_context.option_value(cmd, opt_ss);
+
+  int status = CMD_EXEC_SUCCESS;
+
+  /* If a file is specified, run script mode of the shell, otherwise,  */
+  if (is_cmd_file) {
+    shell->run_script_mode(cmd_ss.c_str(), openfpga_ctx,
+                           cmd_context.option_enable(cmd, opt_batch_mode));
+  } else {
+    /* Split the string with ';' and run each command */
+    /* Remove the space at the end of the line
+     * So that we can check easily if there is a continued line in the end
+     */
+    StringToken cmd_ss_tokenizer(cmd_ss);
+
+    for (std::string cmd_part : cmd_ss_tokenizer.split(";")) {
+      StringToken cmd_part_tokenizer(cmd_part);
+      cmd_part_tokenizer.rtrim(std::string(" "));
+      std::string single_cmd_line = cmd_part_tokenizer.data();
+
+      if (!single_cmd_line.empty()) {
+        status = shell->execute_command(single_cmd_line.c_str(), openfpga_ctx);
+
+        /* Check the execution status of the command,
+         * if fatal error happened, we should abort immediately
+         */
+        if (CMD_EXEC_FATAL_ERROR == status) {
+          return CMD_EXEC_FATAL_ERROR;
+        }
+      }
+    }
+  }
+
+  return CMD_EXEC_SUCCESS;
+}
+
+/********************************************************************
+ * - Add a command to Shell environment: repack
+ * - Add associated options
+ * - Add command dependency
+ *******************************************************************/
+static ShellCommandId add_openfpga_source_command(
+  openfpga::Shell<OpenfpgaContext>& shell,
+  const ShellCommandClassId& cmd_class_id,
+  const std::vector<ShellCommandId>& dependent_cmds) {
+  Command shell_cmd("source");
+
+  /* Add an option '--command_stream' */
+  CommandOptionId opt_cmdstream = shell_cmd.add_option(
+    "command_stream", true,
+    "A string/file stream which contains the commands to be executed");
+  shell_cmd.set_option_require_value(opt_cmdstream, openfpga::OPT_STRING);
+
+  /* Add an option '--from_file' */
+  shell_cmd.add_option("from_file", false,
+                       "Specify the command stream comes from a file");
+
+  /* Add an option '--batch_mode' */
+  shell_cmd.add_option(
+    "batch_mode", false,
+    "Enable batch mode when executing the script from a file (not a string)");
+
+  /* Add command 'repack' to the Shell */
+  ShellCommandId shell_cmd_id = shell.add_command(
+    shell_cmd, "Source a string of commands or execute a script from a file");
+  shell.set_command_class(shell_cmd_id, cmd_class_id);
+  shell.set_command_execute_function(shell_cmd_id, source_existing_command);
+
+  /* Add command dependency to the Shell */
+  shell.set_command_dependency(shell_cmd_id, dependent_cmds);
+
+  return shell_cmd_id;
+}
+
 void add_basic_commands(openfpga::Shell<OpenfpgaContext>& shell) {
   /* Add a new class of commands */
   ShellCommandClassId basic_cmd_class = shell.add_command_class("Basic");
@@ -30,6 +114,10 @@ void add_basic_commands(openfpga::Shell<OpenfpgaContext>& shell) {
   shell.set_command_execute_function(shell_cmd_version_id,
                                      print_openfpga_version_info);
 
+  /* Add 'source' command which can run a set of commands */
+  add_openfpga_source_command(shell, basic_cmd_class,
+                              std::vector<ShellCommandId>());
+
   /* Note:
    * help MUST be the last to add because the linking to execute function will
    * do a snapshot on the shell

openfpga_flow/openfpga_shell_scripts/external_file_for_source_example_script.openfpga

@@ -0,0 +1,27 @@
+# Used in another script 'source_file_example_script.openfpga'
+# Check and correct any naming conflicts in the BLIF netlist
+check_netlist_naming_conflict --fix --report ./netlist_renaming.xml
+
+# Apply fix-up to Look-Up Table truth tables based on packing results
+lut_truth_table_fixup
+
+# Build the module graph
+#  - Enabled compression on routing architecture modules
+#  - Enable pin duplication on grid modules
+build_fabric --compress_routing #--verbose
+
+# Write the fabric hierarchy of module graph to a file
+# This is used by hierarchical PnR flows
+write_fabric_hierarchy --file ./fabric_hierarchy.txt
+
+# Repack the netlist to physical pbs
+# This must be done before bitstream generator and testbench generation
+# Strongly recommend it is done after all the fix-up have been applied
+repack #--verbose
+
+# Build the bitstream
+#  - Output the fabric-independent bitstream to a file
+build_architecture_bitstream --verbose --write_file fabric_independent_bitstream.xml
+
+# Build fabric-dependent bitstream
+build_fabric_bitstream --verbose

openfpga_flow/openfpga_shell_scripts/source_file_example_script.openfpga

@@ -0,0 +1,49 @@
+# Run VPR for the 'and' design
+#--write_rr_graph example_rr_graph.xml
+vpr ${VPR_ARCH_FILE} ${VPR_TESTBENCH_BLIF} --clock_modeling route
+
+# Read OpenFPGA architecture definition
+read_openfpga_arch -f ${OPENFPGA_ARCH_FILE}
+
+# Read OpenFPGA simulation settings
+read_openfpga_simulation_setting -f ${OPENFPGA_SIM_SETTING_FILE}
+
+# Annotate the OpenFPGA architecture to VPR data base
+# to debug use --verbose options
+link_openfpga_arch --activity_file ${ACTIVITY_FILE} --sort_gsb_chan_node_in_edges
+
+# Call a file to run a few commands
+source --command_stream ${OPENFPGA_EXTERNAL_SHELL_SCRIPT_DIR}/external_file_for_source_example_script.openfpga --from_file --batch_mode
+
+# Write fabric-dependent bitstream
+write_fabric_bitstream --file fabric_bitstream.bit --format plain_text
+
+# Write the Verilog netlist for FPGA fabric
+#  - Enable the use of explicit port mapping in Verilog netlist
+write_fabric_verilog --file ./SRC --explicit_port_mapping --include_timing --print_user_defined_template --verbose
+
+# Write the Verilog testbench for FPGA fabric
+#  - We suggest the use of same output directory as fabric Verilog netlists
+#  - Must specify the reference benchmark file if you want to output any testbenches
+#  - Enable top-level testbench which is a full verification including programming circuit and core logic of FPGA
+#  - Enable pre-configured top-level testbench which is a fast verification skipping programming phase
+#  - Simulation ini file is optional and is needed only when you need to interface different HDL simulators using openfpga flow-run scripts
+write_full_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping --include_signal_init --bitstream fabric_bitstream.bit 
+write_preconfigured_fabric_wrapper --embed_bitstream iverilog --file ./SRC  --explicit_port_mapping
+write_preconfigured_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping 
+
+# Write the SDC files for PnR backend
+#  - Turn on every options here
+write_pnr_sdc --file ./SDC
+
+# Write SDC to disable timing for configure ports
+write_sdc_disable_timing_configure_ports --file ./SDC/disable_configure_ports.sdc
+
+# Write the SDC to run timing analysis for a mapped FPGA fabric
+write_analysis_sdc --file ./SDC_analysis
+
+# Finish and exit OpenFPGA
+exit
+
+# Note :
+# To run verification at the end of the flow maintain source in ./SRC directory

openfpga_flow/openfpga_shell_scripts/source_string_example_script.openfpga

@@ -0,0 +1,62 @@
+# Run VPR for the 'and' design
+#--write_rr_graph example_rr_graph.xml
+vpr ${VPR_ARCH_FILE} ${VPR_TESTBENCH_BLIF} --clock_modeling route
+
+# Read OpenFPGA architecture definition
+read_openfpga_arch -f ${OPENFPGA_ARCH_FILE}
+
+# Read OpenFPGA simulation settings
+read_openfpga_simulation_setting -f ${OPENFPGA_SIM_SETTING_FILE}
+
+# Annotate the OpenFPGA architecture to VPR data base
+# to debug use --verbose options
+link_openfpga_arch --activity_file ${ACTIVITY_FILE} --sort_gsb_chan_node_in_edges
+
+# Use a stream  to run a few commands
+# Check and correct any naming conflicts in the BLIF netlist
+# Apply fix-up to Look-Up Table truth tables based on packing results
+# Build the module graph
+#  - Enabled compression on routing architecture modules
+#  - Enable pin duplication on grid modules
+# Write the fabric hierarchy of module graph to a file
+# This is used by hierarchical PnR flows
+# Repack the netlist to physical pbs
+# This must be done before bitstream generator and testbench generation
+# Strongly recommend it is done after all the fix-up have been applied
+# Build the bitstream
+#  - Output the fabric-independent bitstream to a file
+# Build fabric-dependent bitstream
+source --command_stream "check_netlist_naming_conflict --fix --report ./netlist_renaming.xml;lut_truth_table_fixup;build_fabric --compress_routing;write_fabric_hierarchy --file ./fabric_hierarchy.txt;repack;build_architecture_bitstream --verbose --write_file fabric_independent_bitstream.xml;build_fabric_bitstream --verbose"
+
+# Write fabric-dependent bitstream
+write_fabric_bitstream --file fabric_bitstream.bit --format plain_text
+
+# Write the Verilog netlist for FPGA fabric
+#  - Enable the use of explicit port mapping in Verilog netlist
+write_fabric_verilog --file ./SRC --explicit_port_mapping --include_timing --print_user_defined_template --verbose
+
+# Write the Verilog testbench for FPGA fabric
+#  - We suggest the use of same output directory as fabric Verilog netlists
+#  - Must specify the reference benchmark file if you want to output any testbenches
+#  - Enable top-level testbench which is a full verification including programming circuit and core logic of FPGA
+#  - Enable pre-configured top-level testbench which is a fast verification skipping programming phase
+#  - Simulation ini file is optional and is needed only when you need to interface different HDL simulators using openfpga flow-run scripts
+write_full_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping --include_signal_init --bitstream fabric_bitstream.bit 
+write_preconfigured_fabric_wrapper --embed_bitstream iverilog --file ./SRC  --explicit_port_mapping
+write_preconfigured_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping 
+
+# Write the SDC files for PnR backend
+#  - Turn on every options here
+write_pnr_sdc --file ./SDC
+
+# Write SDC to disable timing for configure ports
+write_sdc_disable_timing_configure_ports --file ./SDC/disable_configure_ports.sdc
+
+# Write the SDC to run timing analysis for a mapped FPGA fabric
+write_analysis_sdc --file ./SDC_analysis
+
+# Finish and exit OpenFPGA
+exit
+
+# Note :
+# To run verification at the end of the flow maintain source in ./SRC directory

openfpga_flow/regression_test_scripts/basic_reg_test.sh

@@ -11,6 +11,10 @@ echo -e "Basic regression tests";
 echo -e "Test multiple runs of vpr"
 run-task basic_tests/vpr_standalone $@
 
+echo -e "Test source commands in openfpga shell"
+run-task basic_tests/source_command/source_string $@
+run-task basic_tests/source_command/source_file $@
+
 echo -e "Testing configuration chain of a K4N4 FPGA";
 run-task basic_tests/full_testbench/configuration_chain $@
 run-task basic_tests/full_testbench/configuration_chain_no_time_stamp $@

openfpga_flow/tasks/basic_tests/source_command/source_file/config/task.conf

@@ -0,0 +1,38 @@
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+# Configuration file for running experiments
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+# timeout_each_job : FPGA Task script splits fpga flow into multiple jobs
+# Each job execute fpga_flow script on combination of architecture & benchmark
+# timeout_each_job is timeout for each job
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+
+[GENERAL]
+run_engine=openfpga_shell
+power_tech_file = ${PATH:OPENFPGA_PATH}/openfpga_flow/tech/PTM_45nm/45nm.xml
+power_analysis = true
+spice_output=false
+verilog_output=true
+timeout_each_job = 20*60
+fpga_flow=yosys_vpr
+
+[OpenFPGA_SHELL]
+openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_shell_scripts/source_file_example_script.openfpga
+openfpga_arch_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_arch/k4_N4_40nm_cc_openfpga.xml
+openfpga_sim_setting_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_simulation_settings/auto_sim_openfpga.xml
+openfpga_vpr_device_layout=2x2
+openfpga_vpr_route_chan_width=20
+openfpga_external_shell_script_dir=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_shell_scripts
+
+[ARCHITECTURES]
+arch0=${PATH:OPENFPGA_PATH}/openfpga_flow/vpr_arch/k4_N4_tileable_40nm.xml
+
+[BENCHMARKS]
+bench0=${PATH:OPENFPGA_PATH}/openfpga_flow/benchmarks/micro_benchmark/and2/and2.v
+
+[SYNTHESIS_PARAM]
+bench_read_verilog_options_common = -nolatches
+bench0_top = and2
+
+[SCRIPT_PARAM_MIN_ROUTE_CHAN_WIDTH]
+end_flow_with_test=
+vpr_fpga_verilog_formal_verification_top_netlist=

openfpga_flow/tasks/basic_tests/source_command/source_string/config/task.conf

@@ -0,0 +1,37 @@
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+# Configuration file for running experiments
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+# timeout_each_job : FPGA Task script splits fpga flow into multiple jobs
+# Each job execute fpga_flow script on combination of architecture & benchmark
+# timeout_each_job is timeout for each job
+# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+
+[GENERAL]
+run_engine=openfpga_shell
+power_tech_file = ${PATH:OPENFPGA_PATH}/openfpga_flow/tech/PTM_45nm/45nm.xml
+power_analysis = true
+spice_output=false
+verilog_output=true
+timeout_each_job = 20*60
+fpga_flow=yosys_vpr
+
+[OpenFPGA_SHELL]
+openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_shell_scripts/source_string_example_script.openfpga
+openfpga_arch_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_arch/k4_N4_40nm_cc_openfpga.xml
+openfpga_sim_setting_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_simulation_settings/auto_sim_openfpga.xml
+openfpga_vpr_device_layout=2x2
+openfpga_vpr_route_chan_width=20
+
+[ARCHITECTURES]
+arch0=${PATH:OPENFPGA_PATH}/openfpga_flow/vpr_arch/k4_N4_tileable_40nm.xml
+
+[BENCHMARKS]
+bench0=${PATH:OPENFPGA_PATH}/openfpga_flow/benchmarks/micro_benchmark/and2/and2.v
+
+[SYNTHESIS_PARAM]
+bench_read_verilog_options_common = -nolatches
+bench0_top = and2
+
+[SCRIPT_PARAM_MIN_ROUTE_CHAN_WIDTH]
+end_flow_with_test=
+vpr_fpga_verilog_formal_verification_top_netlist=