Update test flow

libs/libarchopenfpga/src/read_xml_bitstream_setting.cpp

@@ -105,16 +105,18 @@ static void read_xml_non_fabric_bitstream_setting(
 /********************************************************************
  * Parse XML description for a bit setting under a <bit> XML node
  *******************************************************************/
-static void read_xml_bit_setting(
+static void read_xml_overwrite_bit_setting(
-  pugi::xml_node& xml_bit, const pugiutil::loc_data& loc_data,
+  pugi::xml_node& xml_overwrite_bitstream, const pugiutil::loc_data& loc_data,
   openfpga::BitstreamSetting& bitstream_setting) {
-  const std::string& path_attr =
+  for (pugi::xml_node xml_bit : xml_overwrite_bitstream.children()) {
-    get_attribute(xml_bit, "path", loc_data).as_string();
+    const std::string& path_attr =
-  const std::string& value_attr =
+      get_attribute(xml_bit, "path", loc_data).as_string();
-    get_attribute(xml_bit, "value", loc_data).as_string();
+    const std::string& value_attr =
-  VTR_ASSERT_SAFE(value_attr == "0" || value_attr == "1");
+      get_attribute(xml_bit, "value", loc_data).as_string();
-  /* Add to bit */
+    VTR_ASSERT(value_attr == "0" || value_attr == "1");
-  bitstream_setting.add_path_bit_setting(path_attr, value_attr == "1");
+    /* Add to bit */
+    bitstream_setting.add_path_bit_setting(path_attr, value_attr == "1");
+  }
 }
 
 /********************************************************************
@@ -132,9 +134,9 @@ openfpga::BitstreamSetting read_xml_bitstream_setting(
     if ((xml_child.name() != std::string("pb_type")) &&
         (xml_child.name() != std::string("interconnect")) &&
         (xml_child.name() != std::string("non_fabric")) &&
-        (xml_child.name() != std::string("bit"))) {
+        (xml_child.name() != std::string("overwrite_bitstream"))) {
       bad_tag(xml_child, loc_data, Node,
-              {"pb_type | interconnect | non_fabric | bit"});
+              {"pb_type | interconnect | non_fabric | overwrite_bitstream"});
     }
 
     if (xml_child.name() == std::string("pb_type")) {
@@ -147,8 +149,8 @@ openfpga::BitstreamSetting read_xml_bitstream_setting(
       read_xml_non_fabric_bitstream_setting(xml_child, loc_data,
                                             bitstream_setting);
     } else {
-      VTR_ASSERT_SAFE(xml_child.name() == std::string("bit"));
+      VTR_ASSERT_SAFE(xml_child.name() == std::string("overwrite_bitstream"));
-      read_xml_bit_setting(xml_child, loc_data, bitstream_setting);
+      read_xml_overwrite_bit_setting(xml_child, loc_data, bitstream_setting);
     }
   }
 

openfpga/src/base/openfpga_basic.cpp

@@ -68,8 +68,14 @@ int call_external_command(const Command& cmd,
     VTR_LOG("Processer is not available");
     return CMD_EXEC_FATAL_ERROR;
   }
-
+  
-  return system(cmd_ss.c_str());
+  int status = system(cmd_ss.c_str());
+  if (status & 0xFF) {
+    // First 8 bits are system signals
+    return 1;
+  }
+  // real return was actually shifted 8 bits
+  return status >> 8;
 }
 
 } /* end namespace openfpga */

openfpga_flow/regression_test_scripts/fpga_bitstream_reg_test.sh

@@ -15,8 +15,8 @@ echo -e "Testing bitstream generation for an 48x48 FPGA device";
 run-task fpga_bitstream/generate_bitstream/configuration_chain/device_48x48 $@
 run-task fpga_bitstream/generate_bitstream/ql_memory_bank_shift_register/device_48x48 $@
 
-echo -e "Testing bitstream generation for an 4x4 FPGA device (overwrite IO tile bits)";
+echo -e "Testing bitstream generation for an 4x4 FPGA device (randomly overwrite fabric bits)";
-run-task fpga_bitstream/generate_bitstream/configuration_chain/device_4x4_PathBitSetting $@
+run-task fpga_bitstream/overwrite_bitstream/device_4x4 $@
 
 echo -e "Testing bitstream generation for an 96x96 FPGA device";
 run-task fpga_bitstream/generate_bitstream/configuration_chain/device_96x96 $@

openfpga_flow/tasks/fpga_bitstream/generate_bitstream/configuration_chain/device_4x4_PathBitSetting/config/bitstream_annotation.xml

@@ -1,9 +0,0 @@
-<!--
-////////////////////////////////////////////////////////////////////////////////
-# THIS FILE WAS AUTOMATICALLY GENERATED FROM DO NOT EDIT
-////////////////////////////////////////////////////////////////////////////////
--->
-<openfpga_bitstream_setting>
-  <bit value="0" path="fpga_top.grid_io_left_0__1_.logical_tile_io_mode_io__7.logical_tile_io_mode_physical__iopad_0.GPIO_DFFR_mem[0]"/>
-  <bit value="0" path="fpga_top.grid_io_bottom_3__0_.logical_tile_io_mode_io__1.logical_tile_io_mode_physical__iopad_0.GPIO_DFFR_mem[0]"/>
-</openfpga_bitstream_setting>

openfpga_flow/tasks/fpga_bitstream/generate_bitstream/configuration_chain/device_4x4_PathBitSetting/config/write_xml.openfpga

@@ -1,79 +0,0 @@
-# This file is copied from fix_device_example_script.openfpga
-
-# Run VPR for the 'and' design
-#--write_rr_graph example_rr_graph.xml
-vpr ${VPR_ARCH_FILE} ${VPR_TESTBENCH_BLIF} --clock_modeling route --device ${OPENFPGA_VPR_DEVICE_LAYOUT}
-
-# Read OpenFPGA architecture definition
-read_openfpga_arch -f ${OPENFPGA_ARCH_FILE}
-
-# Read OpenFPGA simulation settings
-read_openfpga_simulation_setting -f ${OPENFPGA_SIM_SETTING_FILE}
-
-# Read OpenFPGA bitstream settings
-read_openfpga_bitstream_setting -f ${OPENFPGA_BITSTREAM_SETTING_FILE}
-
-# Annotate the OpenFPGA architecture to VPR data base
-# to debug use --verbose options
-link_openfpga_arch --activity_file ${ACTIVITY_FILE} --sort_gsb_chan_node_in_edges
-
-# Check and correct any naming conflicts in the BLIF netlist
-check_netlist_naming_conflict --fix --report ./netlist_renaming.xml
-
-# Apply fix-up to Look-Up Table truth tables based on packing results
-lut_truth_table_fixup
-
-# Build the module graph
-#  - Enabled compression on routing architecture modules
-#  - Enable pin duplication on grid modules
-build_fabric --compress_routing #--verbose
-
-# Write the fabric hierarchy of module graph to a file
-# This is used by hierarchical PnR flows
-write_fabric_hierarchy --file ./fabric_hierarchy.txt
-
-# Repack the netlist to physical pbs
-# This must be done before bitstream generator and testbench generation
-# Strongly recommend it is done after all the fix-up have been applied
-repack #--verbose
-
-# Build the bitstream
-#  - Output the fabric-independent bitstream to a file
-build_architecture_bitstream --verbose --write_file fabric_independent_bitstream.xml
-
-# Build fabric-dependent bitstream
-build_fabric_bitstream --verbose
-
-# Write fabric-dependent bitstream
-write_fabric_bitstream --file fabric_bitstream.bit --format plain_text
-write_fabric_bitstream --file fabric_bitstream.xml --format xml
-
-# Write the Verilog netlist for FPGA fabric
-#  - Enable the use of explicit port mapping in Verilog netlist
-write_fabric_verilog --file ./SRC --explicit_port_mapping --include_timing --print_user_defined_template --verbose
-
-# Write the Verilog testbench for FPGA fabric
-#  - We suggest the use of same output directory as fabric Verilog netlists
-#  - Must specify the reference benchmark file if you want to output any testbenches
-#  - Enable top-level testbench which is a full verification including programming circuit and core logic of FPGA
-#  - Enable pre-configured top-level testbench which is a fast verification skipping programming phase
-#  - Simulation ini file is optional and is needed only when you need to interface different HDL simulators using openfpga flow-run scripts
-write_full_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping --include_signal_init --bitstream fabric_bitstream.bit
-write_preconfigured_fabric_wrapper --embed_bitstream iverilog --file ./SRC  --explicit_port_mapping
-write_preconfigured_testbench --file ./SRC --reference_benchmark_file_path ${REFERENCE_VERILOG_TESTBENCH} --explicit_port_mapping 
-
-# Write the SDC files for PnR backend
-#  - Turn on every options here
-write_pnr_sdc --file ./SDC
-
-# Write SDC to disable timing for configure ports
-write_sdc_disable_timing_configure_ports --file ./SDC/disable_configure_ports.sdc
-
-# Write the SDC to run timing analysis for a mapped FPGA fabric
-write_analysis_sdc --file ./SDC_analysis
-
-# Finish and exit OpenFPGA
-exit
-
-# Note :
-# To run verification at the end of the flow maintain source in ./SRC directory

openfpga_flow/tasks/fpga_bitstream/overwrite_bitstream/device_4x4/config/task.conf

@@ -16,10 +16,10 @@ timeout_each_job = 20*60
 fpga_flow=vpr_blif
 
 [OpenFPGA_SHELL]
-openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/tasks/fpga_bitstream/generate_bitstream/configuration_chain/device_4x4_PathBitSetting/config/write_xml.openfpga
+openfpga_shell_template=${PATH:OPENFPGA_PATH}/openfpga_flow/tasks/fpga_bitstream/overwrite_bitstream/device_4x4/config/test.openfpga
 openfpga_arch_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_arch/k6_frac_N10_adder_chain_mem16K_40nm_openfpga.xml
 openfpga_sim_setting_file=${PATH:OPENFPGA_PATH}/openfpga_flow/openfpga_simulation_settings/auto_sim_openfpga.xml
-openfpga_bitstream_setting_file=${PATH:OPENFPGA_PATH}/openfpga_flow/tasks/fpga_bitstream/generate_bitstream/configuration_chain/device_4x4_PathBitSetting/config/bitstream_annotation.xml
+openfpga_bitstream_setting_file=bitstream_annotation.xml
 openfpga_vpr_device_layout=4x4
 
 [ARCHITECTURES]

openfpga_flow/tasks/fpga_bitstream/overwrite_bitstream/device_4x4/config/test.openfpga

@@ -0,0 +1,53 @@
+# This file is copied from fix_device_example_script.openfpga
+
+ext_exec --command "python3 ../../../../config/test.py run_golden"
+
+ext_exec --command "python3 ../../../../config/test.py generate_testcase"
+
+# Run VPR for the 'and' design
+#--write_rr_graph example_rr_graph.xml
+vpr ${VPR_ARCH_FILE} ${VPR_TESTBENCH_BLIF} --clock_modeling route --device ${OPENFPGA_VPR_DEVICE_LAYOUT}
+
+# Read OpenFPGA architecture definition
+read_openfpga_arch -f ${OPENFPGA_ARCH_FILE}
+
+# Read OpenFPGA simulation settings
+read_openfpga_simulation_setting -f ${OPENFPGA_SIM_SETTING_FILE}
+
+# Read OpenFPGA bitstream settings
+read_openfpga_bitstream_setting -f ${OPENFPGA_BITSTREAM_SETTING_FILE}
+
+# Annotate the OpenFPGA architecture to VPR data base
+# to debug use --verbose options
+link_openfpga_arch --activity_file ${ACTIVITY_FILE} --sort_gsb_chan_node_in_edges
+
+# Check and correct any naming conflicts in the BLIF netlist
+check_netlist_naming_conflict --fix --report ./netlist_renaming.xml
+
+# Apply fix-up to Look-Up Table truth tables based on packing results
+lut_truth_table_fixup
+
+# Build the module graph
+#  - Enabled compression on routing architecture modules
+#  - Enable pin duplication on grid modules
+build_fabric --compress_routing #--verbose
+
+# Repack the netlist to physical pbs
+# This must be done before bitstream generator and testbench generation
+# Strongly recommend it is done after all the fix-up have been applied
+repack #--verbose
+
+# Build the bitstream
+#  - Output the fabric-independent bitstream to a file
+build_architecture_bitstream --verbose
+
+# Build fabric-dependent bitstream
+build_fabric_bitstream --verbose
+
+# Write fabric-dependent bitstream
+write_fabric_bitstream --file fabric_bitstream.xml --format xml
+
+ext_exec --command "python3 ../../../../config/test.py validate"
+
+# Finish and exit OpenFPGA
+exit

openfpga_flow/tasks/fpga_bitstream/overwrite_bitstream/device_4x4/config/test.py

@@ -0,0 +1,120 @@
+import xml.etree.ElementTree as ET
+import sys
+import shutil
+import os
+import random
+
+random.seed()
+assert len(sys.argv) >= 2
+assert sys.argv[1] in ["run_golden", "generate_testcase", "validate"]
+TEST_BIT_COUNT = 200
+
+def read_fabric_bitstream_xml(file) :
+
+  bit_count = 0
+  tree = ET.parse(file)
+  root = tree.getroot()
+  assert root.tag == "fabric_bitstream", "Root tag is not 'fabric_bitstream', but '%s'" % root.tag
+  for region in root :
+    assert region.tag == "region", "fabric_bitstream child node tag is not 'region', but '%s'" % region.tag
+    for bit in region :
+      assert bit.tag == "bit", "region child node tag is not 'bit', but '%s'" % bit.tag
+      assert "path" in bit.attrib, "Attribute 'path' does not exist in bit node"
+      assert "value" in bit.attrib, "Attribute 'value' does not exist in bit node"
+      assert bit.attrib["value"] in ["0", "1"]
+      bit_count += 1
+  return [tree, bit_count]
+  
+def read_bitstream_annotation_xml(file) :
+  
+  xml = {}
+  tree = ET.parse(file)
+  root = tree.getroot()
+  assert root.tag == "openfpga_bitstream_setting", "Root tag is not 'openfpga_bitstream_setting', but '%s'" % root.tag
+  for overwrite_bitstream in root :
+    assert overwrite_bitstream.tag == "overwrite_bitstream", "openfpga_bitstream_setting child node tag is not 'overwrite_bitstream', but '%s'" % overwrite_bitstream.tag
+    for bit in overwrite_bitstream :
+      assert bit.tag == "bit", "overwrite_bitstream child node tag is not 'bit', but '%s'" % bit.tag
+      assert "path" in bit.attrib, "Attribute 'path' does not exist in bit node"
+      assert "value" in bit.attrib, "Attribute 'value' does not exist in bit node"
+      path = bit.attrib["path"]
+      assert path not in xml
+      index = path.rfind("[")
+      assert index != -1
+      path = "%s.mem_out%s" % (path[:index], path[index:])
+      assert path not in xml
+      assert bit.attrib["value"] in ["0", "1"]
+      xml[path] = bit.attrib["value"]
+  return xml
+
+if sys.argv[1] == "run_golden" :
+
+  openfpga_exe = os.path.abspath("../../../../../../../../../build/openfpga/openfpga")
+  assert os.path.exists(openfpga_exe)
+  shutil.rmtree("golden", ignore_errors=True)
+  os.mkdir("golden")
+  original_openfpga = open("and2_run.openfpga")
+  golden_openfpga = open("golden/and2_run.openfpga", "w")
+  for line in original_openfpga :
+    if line.find("ext_exec") == 0 :
+      pass
+    else :
+      golden_openfpga.write(line)
+  golden_openfpga.close()
+  original_openfpga.close()
+  bitstream_annotation = open("golden/bitstream_annotation.xml", "w")
+  bitstream_annotation.write("<openfpga_bitstream_setting/>\n")
+  bitstream_annotation.close()
+  shutil.copyfile("and2.blif", "golden/and2.blif")
+  shutil.copyfile("and2_ace_out.act", "golden/and2_ace_out.act")
+  cmd = "cd golden && %s -batch -f and2_run.openfpga > golden.log" % (openfpga_exe)
+  assert os.system(cmd) == 0
+
+elif sys.argv[1] == "generate_testcase" :
+  
+  (tree, bit_count) = read_fabric_bitstream_xml("golden/fabric_bitstream.xml")
+  random_bits = []
+  while len(random_bits) != TEST_BIT_COUNT :
+    bit = random.randint(0, bit_count - 1)
+    if bit not in random_bits :
+      random_bits.append(bit)
+  bitstream_annotation = open("bitstream_annotation.xml", "w")
+  bitstream_annotation.write("<openfpga_bitstream_setting>\n")
+  bitstream_annotation.write("  <overwrite_bitstream>\n")
+  index = 0
+  for region in tree.getroot() :
+    for bit in region :
+      if index in random_bits :
+        path = bit.attrib["path"]
+        value = bit.attrib["value"]
+        assert value in ["0", "1"]
+        path = path.replace(".mem_out[", "[")
+        bitstream_annotation.write("    <bit value=\"%s\" path=\"%s\"/>\n" % ("1" if value == "0" else "0", path))
+      index += 1
+  bitstream_annotation.write("  </overwrite_bitstream>\n")
+  bitstream_annotation.write("</openfpga_bitstream_setting>\n")
+  bitstream_annotation.close()
+
+else :
+
+  gtree = ET.parse("golden/fabric_bitstream.xml")
+  tree = ET.parse("fabric_bitstream.xml")
+  bitstream_annotation = read_bitstream_annotation_xml("bitstream_annotation.xml")
+  status = 0
+  checked_count = 0
+  for gregion, region in zip(gtree.getroot(), tree.getroot()) :
+    for gbit, bit in zip(gregion, region) :
+      assert bit.attrib["path"] == gbit.attrib["path"]
+      path = bit.attrib["path"]
+      if path in bitstream_annotation :
+        # This is something we want to overwrite, hence the value should 
+        #   Same in the annotation file
+        #   Not same in golden fabric
+        assert bit.attrib["value"] != gbit.attrib["value"]
+        assert bit.attrib["value"] == bitstream_annotation[path]
+      else :
+        # This is not what we want to overwrite
+        # Hence the value should same in golden fabric
+        assert bit.attrib["value"] == gbit.attrib["value"]
+
+exit(0)