Add optional SEED=n command line option to Makefile, and -S n command line option to test scripts, for deterministic regression tests.

Makefile

@@ -404,16 +404,22 @@ endif
 yosys-abc$(EXE): abc/abc-$(ABCREV)$(EXE)
 	$(P) cp abc/abc-$(ABCREV)$(EXE) yosys-abc$(EXE)
 
+ifneq ($(SEED),)
+SEEDOPT="-S $(SEED)"
+else
+SEEDOPT=""
+endif
+
 test: $(TARGETS) $(EXTRA_TARGETS)
-	+cd tests/simple && bash run-test.sh
+	+cd tests/simple && bash run-test.sh $(SEEDOPT)
-	+cd tests/hana && bash run-test.sh
+	+cd tests/hana && bash run-test.sh $(SEEDOPT)
-	+cd tests/asicworld && bash run-test.sh
+	+cd tests/asicworld && bash run-test.sh $(SEEDOPT)
-	+cd tests/realmath && bash run-test.sh
+	+cd tests/realmath && bash run-test.sh $(SEEDOPT)
-	+cd tests/share && bash run-test.sh
+	+cd tests/share && bash run-test.sh $(SEEDOPT)
-	+cd tests/fsm && bash run-test.sh
+	+cd tests/fsm && bash run-test.sh $(SEEDOPT)
 	+cd tests/techmap && bash run-test.sh
-	+cd tests/memories && bash run-test.sh
+	+cd tests/memories && bash run-test.sh $(SEEDOPT)
-	+cd tests/bram && bash run-test.sh
+	+cd tests/bram && bash run-test.sh $(SEEDOPT)
 	+cd tests/various && bash run-test.sh
 	+cd tests/sat && bash run-test.sh
 	@echo ""

tests/asicworld/run-test.sh

@@ -1,2 +1,14 @@
 #!/bin/bash
-exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-e" *.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-e" *.v

tests/bram/generate.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 import sys
 import random
 
 debug_mode = False
-seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000
 
 def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next):
     while True:
@@ -243,10 +243,23 @@ def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next):
     print("  end", file=tb_f)
     print("endmodule", file=tb_f)
 
-print("Rng seed: %d" % seed)
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 5, help = 'number of test cases to generate')
+parser.add_argument('-d', '--debug', action='store_true')
+args = parser.parse_args()
+
+debug_mode = args.debug
+
+if args.seed is not None:
+    seed = args.seed
+else:
+    seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000
+
+print("PRNG seed: %d" % seed)
 random.seed(seed)
 
-for k1 in range(5):
+for k1 in range(args.count):
     dsc_f = open("temp/brams_%02d.txt" % k1, "w")
     sim_f = open("temp/brams_%02d.v" % k1, "w")
     ref_f = open("temp/brams_%02d_ref.v" % k1, "w")

tests/bram/run-test.sh

@@ -4,11 +4,26 @@
 # MAKE="make -j8" time bash -c 'for ((i=0; i<100; i++)); do echo "-- $i --"; bash run-test.sh || exit 1; done'
 
 set -e
+
+OPTIND=1
+count=5
+seed=""    # default to no seed specified
+debug=""
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	d) debug="-d" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 
 echo "generating tests.."
-python3 generate.py
+python3 generate.py $debug -c $count $seed
 
 {
 	echo -n "all:"

tests/fsm/generate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -30,7 +31,16 @@ def random_expr(variables):
         return "%d'd%s" % (bits, random.randint(0, 2**bits-1))
     raise AssertionError
 
-for idx in range(50):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 50, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             rst2 = random.choice([False, True])

tests/fsm/run-test.sh

@@ -5,10 +5,22 @@
 
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 {
 	all_targets="all_targets:"

tests/hana/run-test.sh

@@ -1,2 +1,14 @@
 #!/bin/bash
-exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v

tests/memories/run-test.sh

@@ -1,7 +1,18 @@
 #!/bin/bash
 
 set -e
-bash ../tools/autotest.sh -G *.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+bash ../tools/autotest.sh $seed -G *.v
 
 for f in `egrep -l 'expect-(wr|rd)-ports' *.v`; do
 	echo -n "Testing expectations for $f .."

tests/realmath/generate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -36,7 +37,16 @@ def random_expression(depth = 3, maxparam = 0):
         return op + '(' + recursion() + ', ' + recursion() + ')'
     raise
 
-for idx in range(100):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             print('module uut_%05d(output [63:0] %s);\n' % (idx, ', '.join(['y%02d' % i for i in range(100)])))

tests/realmath/run-test.sh

@@ -1,14 +1,26 @@
 #!/bin/bash
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 cd temp
 echo "running tests.."
-for ((i = 0; i < 100; i++)); do
+for ((i = 0; i < $count; i++)); do
 	echo -n "[$i]"
 	idx=$( printf "%05d" $i )
 	../../../yosys -qq uut_${idx}.ys

tests/share/generate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -21,7 +22,16 @@ def maybe_plus_x(expr):
     else:
         return expr
 
-for idx in range(100):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             if random.choice(['bin', 'uni']) == 'bin':

tests/share/run-test.sh

@@ -5,10 +5,22 @@
 
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 echo "running tests.."
 for i in $( ls temp/*.ys | sed 's,[^0-9],,g; s,^0*\(.\),\1,g;' ); do

tests/simple/run-test.sh

@@ -1,9 +1,20 @@
 #!/bin/bash
 
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 # check for Icarus Verilog
 if ! which iverilog > /dev/null ; then
   echo "$0: Error: Icarus Verilog 'iverilog' not found."
   exit 1
 fi
 
-exec ${MAKE:-make} -f ../tools/autotest.mk *.v
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed *.v

tests/tools/autotest.mk

@@ -1,8 +1,13 @@
 
 EXTRA_FLAGS=
+SEED=
+
+ifneq ($(strip $(SEED)),)
+SEEDOPT=-S$(SEED)
+endif
 
 $(MAKECMDGOALS):
-	@$(basename $(MAKEFILE_LIST)).sh -G -j $(EXTRA_FLAGS) $@
+	@$(basename $(MAKEFILE_LIST)).sh -G -j $(SEEDOPT) $(EXTRA_FLAGS) $@
 
 .PHONY: $(MAKECMDGOALS)