Merge pull request #2115 from whitequark/cxxrtl-introspection

cxxrtl: add debug information to the C++ API, and add introspection via a new C API

backends/cxxrtl/cxxrtl.cc

@@ -502,6 +502,15 @@ std::string escape_cxx_string(const std::string &input)
 	return output;
 }
 
+template<class T>
+std::string get_hdl_name(T *object)
+{
+	if (object->has_attribute(ID::hdlname))
+		return object->get_string_attribute(ID::hdlname);
+	else
+		return object->name.str();
+}
+
 struct CxxrtlWorker {
 	bool split_intf = false;
 	std::string intf_filename;
@@ -516,6 +525,8 @@ struct CxxrtlWorker {
 	bool run_proc_flatten = false;
 	bool max_opt_level = false;
 
+	bool debug_info = false;
+
 	std::ostringstream f;
 	std::string indent;
 	int temporary = 0;
@@ -1593,6 +1604,34 @@ struct CxxrtlWorker {
 		dec_indent();
 	}
 
+	void dump_debug_info_method(RTLIL::Module *module)
+	{
+		inc_indent();
+			f << indent << "assert(path.empty() || path[path.size() - 1] == ' ');\n";
+			for (auto wire : module->wires()) {
+				if (wire->name[0] != '\\')
+					continue;
+				if (localized_wires.count(wire))
+					continue;
+				f << indent << "items.emplace(path + " << escape_cxx_string(get_hdl_name(wire));
+				f << ", debug_item(" << mangle(wire) << "));\n";
+			}
+			for (auto &memory_it : module->memories) {
+				if (memory_it.first[0] != '\\')
+					continue;
+				f << indent << "items.emplace(path + " << escape_cxx_string(get_hdl_name(memory_it.second));
+				f << ", debug_item(" << mangle(memory_it.second) << "));\n";
+			}
+			for (auto cell : module->cells()) {
+				if (is_internal_cell(cell->type))
+					continue;
+				const char *access = is_cxxrtl_blackbox_cell(cell) ? "->" : ".";
+				f << indent << mangle(cell) << access << "debug_info(items, ";
+				f << "path + " << escape_cxx_string(get_hdl_name(cell) + ' ') << ");\n";
+			}
+		dec_indent();
+	}
+
 	void dump_metadata_map(const dict<RTLIL::IdString, RTLIL::Const> &metadata_map)
 	{
 		if (metadata_map.empty()) {
@@ -1641,6 +1680,12 @@ struct CxxrtlWorker {
 				dump_commit_method(module);
 				f << indent << "}\n";
 				f << "\n";
+				if (debug_info) {
+					f << indent << "void debug_info(debug_items &items, std::string path = \"\") override {\n";
+					dump_debug_info_method(module);
+					f << indent << "}\n";
+					f << "\n";
+				}
 				f << indent << "static std::unique_ptr<" << mangle(module);
 				f << template_params(module, /*is_decl=*/false) << "> ";
 				f << "create(std::string name, metadata_map parameters, metadata_map attributes);\n";
@@ -1689,7 +1734,7 @@ struct CxxrtlWorker {
 					if (cell_module->get_bool_attribute(ID(cxxrtl_blackbox))) {
 						f << indent << "std::unique_ptr<" << mangle(cell_module) << template_args(cell) << "> ";
 						f << mangle(cell) << " = " << mangle(cell_module) << template_args(cell);
-						f << "::create(" << escape_cxx_string(cell->name.str()) << ", ";
+						f << "::create(" << escape_cxx_string(get_hdl_name(cell)) << ", ";
 						dump_metadata_map(cell->parameters);
 						f << ", ";
 						dump_metadata_map(cell->attributes);
@@ -1703,6 +1748,8 @@ struct CxxrtlWorker {
 					f << "\n";
 				f << indent << "bool eval() override;\n";
 				f << indent << "bool commit() override;\n";
+				if (debug_info)
+					f << indent << "void debug_info(debug_items &items, std::string path = \"\") override;\n";
 			dec_indent();
 			f << indent << "}; // struct " << mangle(module) << "\n";
 			f << "\n";
@@ -1721,10 +1768,17 @@ struct CxxrtlWorker {
 		dump_commit_method(module);
 		f << indent << "}\n";
 		f << "\n";
+		if (debug_info) {
+			f << indent << "void " << mangle(module) << "::debug_info(debug_items &items, std::string path) {\n";
+			dump_debug_info_method(module);
+			f << indent << "}\n";
+			f << "\n";
+		}
 	}
 
 	void dump_design(RTLIL::Design *design)
 	{
+		RTLIL::Module *top_module = nullptr;
 		std::vector<RTLIL::Module*> modules;
 		TopoSort<RTLIL::Module*> topo_design;
 		for (auto module : design->modules()) {
@@ -1734,6 +1788,8 @@ struct CxxrtlWorker {
 				modules.push_back(module); // cxxrtl blackboxes first
 			if (module->get_blackbox_attribute() || module->get_bool_attribute(ID(cxxrtl_blackbox)))
 				continue;
+			if (module->get_bool_attribute(ID::top))
+				top_module = module;
 
 			topo_design.node(module);
 			for (auto cell : module->cells()) {
@@ -1755,6 +1811,25 @@ struct CxxrtlWorker {
 			f << "#ifndef " << include_guard << "\n";
 			f << "#define " << include_guard << "\n";
 			f << "\n";
+			if (top_module != nullptr && debug_info) {
+				f << "#include <backends/cxxrtl/cxxrtl_capi.h>\n";
+				f << "\n";
+				f << "#ifdef __cplusplus\n";
+				f << "extern \"C\" {\n";
+				f << "#endif\n";
+				f << "\n";
+				f << "cxxrtl_toplevel " << design_ns << "_create();\n";
+				f << "\n";
+				f << "#ifdef __cplusplus\n";
+				f << "}\n";
+				f << "#endif\n";
+				f << "\n";
+			} else {
+				f << "// The CXXRTL C API is not available because the design is built without debug information.\n";
+				f << "\n";
+			}
+			f << "#ifdef __cplusplus\n";
+			f << "\n";
 			f << "#include <backends/cxxrtl/cxxrtl.h>\n";
 			f << "\n";
 			f << "using namespace cxxrtl;\n";
@@ -1765,6 +1840,8 @@ struct CxxrtlWorker {
 				dump_module_intf(module);
 			f << "} // namespace " << design_ns << "\n";
 			f << "\n";
+			f << "#endif // __cplusplus\n";
+			f << "\n";
 			f << "#endif\n";
 			*intf_f << f.str(); f.str("");
 		}
@@ -1774,6 +1851,10 @@ struct CxxrtlWorker {
 		else
 			f << "#include <backends/cxxrtl/cxxrtl.h>\n";
 		f << "\n";
+		f << "#ifdef CXXRTL_INCLUDE_CAPI_IMPL\n";
+		f << "#include <backends/cxxrtl/cxxrtl_capi.cc>\n";
+		f << "#endif\n";
+		f << "\n";
 		f << "using namespace cxxrtl_yosys;\n";
 		f << "\n";
 		f << "namespace " << design_ns << " {\n";
@@ -1784,6 +1865,17 @@ struct CxxrtlWorker {
 			dump_module_impl(module);
 		}
 		f << "} // namespace " << design_ns << "\n";
+		f << "\n";
+		if (top_module != nullptr && debug_info) {
+			f << "cxxrtl_toplevel " << design_ns << "_create() {\n";
+			inc_indent();
+				f << indent << "return new _cxxrtl_toplevel { ";
+				f << "std::make_unique<" << design_ns << "::" << mangle(top_module) << ">()";
+				f << " };\n";
+			dec_indent();
+			f << "}\n";
+		}
+
 		*impl_f << f.str(); f.str("");
 	}
 
@@ -2120,6 +2212,7 @@ struct CxxrtlWorker {
 
 struct CxxrtlBackend : public Backend {
 	static const int DEFAULT_OPT_LEVEL = 5;
+	static const int DEFAULT_DEBUG_LEVEL = 1;
 
 	CxxrtlBackend() : Backend("cxxrtl", "convert design to C++ RTL simulation") { }
 	void help() YS_OVERRIDE
@@ -2313,10 +2406,22 @@ struct CxxrtlBackend : public Backend {
 		log("    -O5\n");
 		log("        like -O4, and run `proc; flatten` first.\n");
 		log("\n");
+		log("    -g <level>\n");
+		log("        set the debug level. the default is -g%d. higher debug levels provide\n", DEFAULT_DEBUG_LEVEL);
+		log("        more visibility and generate more code, but do not pessimize evaluation.\n");
+		log("\n");
+		log("    -g0\n");
+		log("        no debug information.\n");
+		log("\n");
+		log("    -g1\n");
+		log("        debug information for non-localized public wires.\n");
+		log("\n");
 	}
+
 	void execute(std::ostream *&f, std::string filename, std::vector<std::string> args, RTLIL::Design *design) YS_OVERRIDE
 	{
 		int opt_level = DEFAULT_OPT_LEVEL;
+		int debug_level = DEFAULT_DEBUG_LEVEL;
 		CxxrtlWorker worker;
 
 		log_header(design, "Executing CXXRTL backend.\n");
@@ -2332,6 +2437,14 @@ struct CxxrtlBackend : public Backend {
 				opt_level = std::stoi(args[argidx].substr(2));
 				continue;
 			}
+			if (args[argidx] == "-g" && argidx+1 < args.size()) {
+				debug_level = std::stoi(args[++argidx]);
+				continue;
+			}
+			if (args[argidx].substr(0, 2) == "-g" && args[argidx].size() == 3 && isdigit(args[argidx][2])) {
+				debug_level = std::stoi(args[argidx].substr(2));
+				continue;
+			}
 			if (args[argidx] == "-header") {
 				worker.split_intf = true;
 				continue;
@@ -2368,6 +2481,17 @@ struct CxxrtlBackend : public Backend {
 				log_cmd_error("Invalid optimization level %d.\n", opt_level);
 		}
 
+		switch (debug_level) {
+			// the highest level here must match DEFAULT_DEBUG_LEVEL
+			case 1:
+				worker.debug_info = true;
+				YS_FALLTHROUGH
+			case 0:
+				break;
+			default:
+				log_cmd_error("Invalid optimization level %d.\n", opt_level);
+		}
+
 		std::ofstream intf_f;
 		if (worker.split_intf) {
 			if (filename == "<stdout>")

backends/cxxrtl/cxxrtl.h

@@ -33,13 +33,15 @@
 #include <memory>
 #include <sstream>
 
-// The cxxrtl support library implements compile time specialized arbitrary width arithmetics, as well as provides
+#include <backends/cxxrtl/cxxrtl_capi.h>
+
+// The CXXRTL support library implements compile time specialized arbitrary width arithmetics, as well as provides
 // composite lvalues made out of bit slices and concatenations of lvalues. This allows the `write_cxxrtl` pass
 // to perform a straightforward translation of RTLIL structures to readable C++, relying on the C++ compiler
 // to unwrap the abstraction and generate efficient code.
 namespace cxxrtl {
 
-// All arbitrary-width values in cxxrtl are backed by arrays of unsigned integers called chunks. The chunk size
+// All arbitrary-width values in CXXRTL are backed by arrays of unsigned integers called chunks. The chunk size
 // is the same regardless of the value width to simplify manipulating values via FFI interfaces, e.g. driving
 // and introspecting the simulation in Python.
 //
@@ -49,6 +51,8 @@ namespace cxxrtl {
 // invisible to the compiler, (b) we often operate on non-power-of-2 values and have to clear the high bits anyway.
 // Therefore, using relatively wide chunks and clearing the high bits explicitly and only when we know they may be
 // clobbered results in simpler generated code.
+typedef uint32_t chunk_t;
+
 template<typename T>
 struct chunk_traits {
 	static_assert(std::is_integral<T>::value && std::is_unsigned<T>::value,
@@ -65,7 +69,7 @@ template<size_t Bits>
 struct value : public expr_base<value<Bits>> {
 	static constexpr size_t bits = Bits;
 
-	using chunk = chunk_traits<uint32_t>;
+	using chunk = chunk_traits<chunk_t>;
 	static constexpr chunk::type msb_mask = (Bits % chunk::bits == 0) ? chunk::mask
 		: chunk::mask >> (chunk::bits - (Bits % chunk::bits));
 
@@ -712,6 +716,56 @@ struct metadata {
 
 typedef std::map<std::string, metadata> metadata_map;
 
+// This structure is intended for consumption via foreign function interfaces, like Python's ctypes.
+// Because of this it uses a C-style layout that is easy to parse rather than more idiomatic C++.
+//
+// To avoid violating strict aliasing rules, this structure has to be a subclass of the one used
+// in the C API, or it would not be possible to cast between the pointers to these.
+struct debug_item : ::cxxrtl_object {
+	enum : uint32_t {
+		VALUE  = CXXRTL_VALUE,
+		WIRE   = CXXRTL_WIRE,
+		MEMORY = CXXRTL_MEMORY,
+	};
+
+	template<size_t Bits>
+	debug_item(value<Bits> &item) {
+		static_assert(sizeof(item) == value<Bits>::chunks * sizeof(chunk_t),
+		              "value<Bits> is not compatible with C layout");
+		type  = VALUE;
+		width = Bits;
+		depth = 1;
+		curr  = item.data;
+		next  = item.data;
+	}
+
+	template<size_t Bits>
+	debug_item(wire<Bits> &item) {
+		static_assert(sizeof(item.curr) == value<Bits>::chunks * sizeof(chunk_t) &&
+		              sizeof(item.next) == value<Bits>::chunks * sizeof(chunk_t),
+		              "wire<Bits> is not compatible with C layout");
+		type  = WIRE;
+		width = Bits;
+		depth = 1;
+		curr  = item.curr.data;
+		next  = item.next.data;
+	}
+
+	template<size_t Width>
+	debug_item(memory<Width> &item) {
+		static_assert(sizeof(item.data[0]) == value<Width>::chunks * sizeof(chunk_t),
+		              "memory<Width> is not compatible with C layout");
+		type  = MEMORY;
+		width = Width;
+		depth = item.data.size();
+		curr  = item.data.empty() ? nullptr : item.data[0].data;
+		next  = nullptr;
+	}
+};
+static_assert(std::is_standard_layout<debug_item>::value, "debug_item is not compatible with C layout");
+
+typedef std::map<std::string, debug_item> debug_items;
+
 struct module {
 	module() {}
 	virtual ~module() {}
@@ -731,11 +785,18 @@ struct module {
 		} while (commit() && !converged);
 		return deltas;
 	}
+
+	virtual void debug_info(debug_items &items, std::string path = "") {}
 };
 
 } // namespace cxxrtl
 
-// Definitions of internal Yosys cells. Other than the functions in this namespace, cxxrtl is fully generic
+// Internal structure used to communicate with the implementation of the C interface.
+typedef struct _cxxrtl_toplevel {
+	std::unique_ptr<cxxrtl::module> module;
+} *cxxrtl_toplevel;
+
+// Definitions of internal Yosys cells. Other than the functions in this namespace, CXXRTL is fully generic
 // and indepenent of Yosys implementation details.
 //
 // The `write_cxxrtl` pass translates internal cells (cells with names that start with `$`) to calls of these

backends/cxxrtl/cxxrtl_capi.cc

@@ -0,0 +1,55 @@
+/*
+ *  yosys -- Yosys Open SYnthesis Suite
+ *
+ *  Copyright (C) 2020  whitequark <whitequark@whitequark.org>
+ *
+ *  Permission to use, copy, modify, and/or distribute this software for any
+ *  purpose with or without fee is hereby granted.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ */
+
+// This file is a part of the CXXRTL C API. It should be used together with `cxxrtl_capi.h`.
+
+#include <backends/cxxrtl/cxxrtl.h>
+#include <backends/cxxrtl/cxxrtl_capi.h>
+
+struct _cxxrtl_handle {
+	std::unique_ptr<cxxrtl::module> module;
+	cxxrtl::debug_items objects;
+};
+
+cxxrtl_handle cxxrtl_create(cxxrtl_toplevel design) {
+	cxxrtl_handle handle = new _cxxrtl_handle;
+	handle->module = std::move(design->module);
+	handle->module->debug_info(handle->objects);
+	delete design;
+	return handle;
+}
+
+void cxxrtl_destroy(cxxrtl_handle handle) {
+	delete handle;
+}
+
+size_t cxxrtl_step(cxxrtl_handle handle) {
+	return handle->module->step();
+}
+
+cxxrtl_object *cxxrtl_get(cxxrtl_handle handle, const char *name) {
+	if (handle->objects.count(name) > 0)
+		return static_cast<cxxrtl_object*>(&handle->objects.at(name));
+	return nullptr;
+}
+
+void cxxrtl_enum(cxxrtl_handle handle, void *data,
+                 void (*callback)(void *data, const char *name, struct cxxrtl_object *object)) {
+	for (auto &it : handle->objects)
+		callback(data, it.first.c_str(), static_cast<cxxrtl_object*>(&it.second));
+}

backends/cxxrtl/cxxrtl_capi.h

@@ -0,0 +1,151 @@
+/*
+ *  yosys -- Yosys Open SYnthesis Suite
+ *
+ *  Copyright (C) 2020  whitequark <whitequark@whitequark.org>
+ *
+ *  Permission to use, copy, modify, and/or distribute this software for any
+ *  purpose with or without fee is hereby granted.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ */
+
+#ifndef CXXRTL_CAPI_H
+#define CXXRTL_CAPI_H
+
+// This file is a part of the CXXRTL C API. It should be used together with `cxxrtl_capi.cc`.
+//
+// The CXXRTL C API makes it possible to drive CXXRTL designs using C or any other language that
+// supports the C ABI, for example, Python. It does not provide a way to implement black boxes.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Opaque reference to a design toplevel.
+//
+// A design toplevel can only be used to create a design handle.
+typedef struct _cxxrtl_toplevel *cxxrtl_toplevel;
+
+// The constructor for a design toplevel is provided as a part of generated code for that design.
+// Its prototype matches:
+//
+// cxxrtl_toplevel <design-name>_create();
+
+// Opaque reference to a design handle.
+//
+// A design handle is required by all operations in the C API.
+typedef struct _cxxrtl_handle *cxxrtl_handle;
+
+// Create a design handle from a design toplevel.
+//
+// The `design` is consumed by this operation and cannot be used afterwards.
+cxxrtl_handle cxxrtl_create(cxxrtl_toplevel design);
+
+// Release all resources used by a design and its handle.
+void cxxrtl_destroy(cxxrtl_handle handle);
+
+// Simulate the design to a fixed point.
+//
+// Returns the number of delta cycles.
+size_t cxxrtl_step(cxxrtl_handle handle);
+
+// Type of a simulated object.
+enum cxxrtl_type {
+	// Values correspond to singly buffered netlist nodes, i.e. nodes driven exclusively by
+	// combinatorial cells, or toplevel input nodes.
+	//
+	// Values can be inspected via the `curr` pointer and modified via the `next` pointer (which are
+	// equal for values); however, note that changes to the bits driven by combinatorial cells will
+	// be ignored.
+	//
+	// Values always have depth 1.
+	CXXRTL_VALUE = 0,
+
+	// Wires correspond to doubly buffered netlist nodes, i.e. nodes driven, at least in part, by
+	// storage cells, or by combinatorial cells that are a part of a feedback path.
+	//
+	// Wires can be inspected via the `curr` pointer and modified via the `next` pointer (which are
+	// distinct for wires); however, note that changes to the bits driven by combinatorial cells will
+	// be ignored.
+	//
+	// Wires always have depth 1.
+	CXXRTL_WIRE = 1,
+
+	// Memories correspond to memory cells.
+	//
+	// Memories can be inspected and modified via the `curr` pointer. Due to a limitation of this
+	// API, memories cannot yet be modified in a guaranteed race-free way, and the `next` pointer is
+	// always NULL.
+	CXXRTL_MEMORY = 2,
+
+	// More object types will be added in the future, but the existing ones will never change.
+};
+
+// Description of a simulated object.
+//
+// The `data` array can be accessed directly to inspect and, if applicable, modify the bits
+// stored in the object.
+struct cxxrtl_object {
+	// Type of the object.
+	//
+	// All objects have the same memory layout determined by `width` and `depth`, but the type
+	// determines all other properties of the object.
+	uint32_t type; // actually `enum cxxrtl_type`
+
+	// Width of the object in bits.
+	size_t width;
+
+	// Depth of the object. Only meaningful for memories; for other objects, always 1.
+	size_t depth;
+
+	// Bits stored in the object, as 32-bit chunks, least significant bits first.
+	//
+	// The width is rounded up to a multiple of 32; the padding bits are always set to 0 by
+	// the simulation code, and must be always written as 0 when modified by user code.
+	// In memories, every element is stored contiguously. Therefore, the total number of chunks
+	// in any object is `((width + 31) / 32) * depth`.
+	//
+	// To allow the simulation to be partitioned into multiple independent units communicating
+	// through wires, the bits are double buffered. To avoid race conditions, user code should
+	// always read from `curr` and write to `next`. The `curr` pointer is always valid; for objects
+	// that cannot be modified, or cannot be modified in a race-free way, `next` is NULL.
+	uint32_t *curr;
+	uint32_t *next;
+
+	// More description fields will be added in the future, but the existing ones will never change.
+};
+
+// Retrieve description of a simulated object.
+//
+// The `name` is the full hierarchical name of the object in the Yosys notation, where public names
+// have a `\` prefix and hierarchy levels are separated by single spaces. For example, if
+// the top-level module instantiates a module `foo`, which in turn contains a wire `bar`, the full
+// hierarchical name is `\foo \bar`.
+//
+// Returns the object if it was found, NULL otherwise. The returned value is valid until the design
+// is destroyed.
+struct cxxrtl_object *cxxrtl_get(cxxrtl_handle handle, const char *name);
+
+// Enumerate simulated objects.
+//
+// For every object in the simulation, `callback` is called with the provided `data`, the full
+// hierarchical name of the object (see `cxxrtl_get` for details), and the object description.
+// The provided `name` and `object` values are valid until the design is destroyed.
+void cxxrtl_enum(cxxrtl_handle handle, void *data,
+                 void (*callback)(void *data, const char *name, struct cxxrtl_object *object));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif